Update knowledge and skill wizard info and doc upload pages

src/app/api/github/git-info/route.ts

@@ -0,0 +1,105 @@
+// src/app/api/github/knowledge-files/route.ts
+import { NextRequest, NextResponse } from 'next/server';
+import { getToken } from 'next-auth/jwt';
+import {
+  BASE_BRANCH,
+  checkIfRepoExists,
+  fetchCommitInfo,
+  fetchMarkdownFiles,
+  forkRepo,
+  getBranchSha,
+  GITHUB_API_URL,
+  TAXONOMY_DOCUMENTS_REPO
+} from '@/app/api/github/utils';
+
+export async function GET(req: NextRequest) {
+  const token = await getToken({ req, secret: process.env.NEXTAUTH_SECRET! });
+
+  if (!token || !token.accessToken) {
+    console.error('Unauthorized: Missing or invalid access token');
+    return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
+  }
+
+  const githubToken = token.accessToken as string;
+  const headers = {
+    'Content-Type': 'application/json',
+    Authorization: `Bearer ${githubToken}`,
+    Accept: 'application/vnd.github+json',
+    'X-GitHub-Api-Version': '2022-11-28'
+  };
+
+  try {
+    // Fetch GitHub username
+    const githubUsername = await getGitHubUsername(headers);
+
+    // Split the TAXONOMY_DOCUMENTS_REPO into owner and repo name
+    const repoPath = TAXONOMY_DOCUMENTS_REPO.replace('github.com/', '');
+    const [repoOwner, repoName] = repoPath.split('/');
+
+    // Check if the repository is already forked
+    const repoForked = await checkIfRepoExists(headers, githubUsername, repoName);
+    console.log(`Repository forked: ${repoForked}`);
+    if (!repoForked) {
+      // Fork the repository if it is not already forked
+      await forkRepo(headers, repoOwner, repoName, githubUsername);
+      // Add a delay to ensure the fork operation completes to avoid a race condition when retrieving the bas SHA
+      // This only occurs if this is the first time submitting and the fork isn't present.
+      // TODO change to a retry
+      console.log('Pause 5s for the forking operation to complete');
+      await new Promise((resolve) => setTimeout(resolve, 5000));
+      console.log('Repository forked');
+    }
+
+    // Fetch the latest commit SHA of the base branch
+    const baseBranchSha = await getBranchSha(headers, githubUsername, repoName, BASE_BRANCH);
+    console.log(`Base branch SHA: ${baseBranchSha}`);
+
+    const files = await fetchMarkdownFiles(headers, githubUsername, repoName, BASE_BRANCH);
+
+    let mostRecentSha = '';
+    let mostRecentDate = 0;
+    let mostRecentFiles: string[] = [];
+
+    for (const file of files) {
+      const commitInfo = await fetchCommitInfo(headers, githubUsername, repoName, file.path);
+      if (commitInfo) {
+        const { sha, date } = commitInfo;
+        const commitDate = new Date(date).getTime();
+        if (commitDate > mostRecentDate) {
+          mostRecentDate = commitDate;
+          mostRecentSha = sha;
+          mostRecentFiles = [];
+        }
+        if (sha === mostRecentSha) {
+          mostRecentFiles.push(file.path);
+        }
+      }
+    }
+    const fileNames = mostRecentFiles.join(',');
+
+    return NextResponse.json(
+      {
+        repoUrl: `https://github.com/${githubUsername}/${repoName}`,
+        commitSha: baseBranchSha,
+        fileNames
+      },
+      { status: 201 }
+    );
+  } catch (error) {
+    console.error('Failed to retrieve document info:', error);
+    return NextResponse.json({ error: 'Failed to retrieve document info' }, { status: 500 });
+  }
+}
+
+async function getGitHubUsername(headers: HeadersInit): Promise<string> {
+  const response = await fetch(`${GITHUB_API_URL}/user`, { headers });
+
+  if (!response.ok) {
+    const errorText = await response.text();
+    console.error('Failed to fetch GitHub username:', response.status, errorText);
+    throw new Error('Failed to fetch GitHub username');
+  }
+
+  const data = await response.json();
+  return data.login;
+}

src/app/api/github/knowledge-files/route.ts

@@ -1,10 +1,16 @@
 // src/app/api/github/knowledge-files/route.ts
 import { NextRequest, NextResponse } from 'next/server';
 import { getToken } from 'next-auth/jwt';
-
-const GITHUB_API_URL = 'https://api.github.com';
-const TAXONOMY_DOCUMENTS_REPO = process.env.NEXT_PUBLIC_TAXONOMY_DOCUMENTS_REPO || 'https://github.com/instructlab-public/taxonomy-knowledge-docs';
-const BASE_BRANCH = 'main';
+import {
+  BASE_BRANCH,
+  checkIfRepoExists,
+  fetchCommitInfo,
+  fetchMarkdownFiles,
+  forkRepo,
+  getBranchSha,
+  GITHUB_API_URL,
+  TAXONOMY_DOCUMENTS_REPO
+} from '@/app/api/github/utils';
 
 // Interface for the response
 interface KnowledgeFile {
@@ -100,62 +106,6 @@ async function getGitHubUsernameAndEmail(headers: HeadersInit): Promise<{ github
   return { githubUsername: data.login, userEmail: data.email };
 }
 
-async function checkIfRepoExists(headers: HeadersInit, owner: string, repo: string): Promise<boolean> {
-  const response = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}`, { headers });
-  const exists = response.ok;
-  if (!exists) {
-    const errorText = await response.text();
-    console.error('Repository does not exist:', response.status, errorText);
-  }
-  return exists;
-}
-
-async function forkRepo(headers: HeadersInit, owner: string, repo: string, forkOwner: string) {
-  const response = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/forks`, {
-    method: 'POST',
-    headers
-  });
-
-  if (!response.ok) {
-    const errorText = await response.text();
-    console.error('Failed to fork repository:', response.status, errorText);
-    throw new Error('Failed to fork repository');
-  }
-
-  // Wait for the fork to be created
-  let forkCreated = false;
-  for (let i = 0; i < 10; i++) {
-    const forkExists = await checkIfRepoExists(headers, forkOwner, repo);
-    if (forkExists) {
-      forkCreated = true;
-      break;
-    }
-    await new Promise((resolve) => setTimeout(resolve, 3000));
-  }
-
-  if (!forkCreated) {
-    throw new Error('Failed to confirm fork creation');
-  }
-}
-
-async function getBranchSha(headers: HeadersInit, owner: string, repo: string, branch: string): Promise<string> {
-  console.log(`Fetching branch SHA for ${branch}...`);
-  const response = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/git/ref/heads/${branch}`, { headers });
-
-  if (!response.ok) {
-    const errorText = await response.text();
-    console.error('Failed to get branch SHA:', response.status, errorText);
-    if (response.status === 409 && errorText.includes('Git Repository is empty')) {
-      throw new Error('Git Repository is empty.');
-    }
-    throw new Error('Failed to get branch SHA');
-  }
-
-  const data = await response.json();
-  console.log('Branch SHA:', data.object.sha);
-  return data.object.sha;
-}
-
 async function createFilesCommit(
   headers: HeadersInit,
   owner: string,
@@ -292,55 +242,6 @@ export async function GET(req: NextRequest) {
   }
 }
 
-// Fetch all markdown files from the main branch
-async function fetchMarkdownFiles(
-  headers: HeadersInit,
-  owner: string,
-  repo: string,
-  branchName: string
-): Promise<{ path: string; content: string }[]> {
-  try {
-    const response = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/git/trees/${branchName}?recursive=1`, { headers });
-    if (!response.ok) {
-      const errorText = await response.text();
-      console.error('Failed to fetch files from knowledge document repository:', response.status, errorText);
-      throw new Error('Failed to fetch file from knowledge document  repository:');
-    }
-
-    const data = await response.json();
-    const files = data.tree.filter(
-      (item: { type: string; path: string }) => item.type === 'blob' && item.path.endsWith('.md') && item.path !== 'README.md'
-    );
-    return files.map((file: { path: string; content: string }) => ({ path: file.path, content: file.content }));
-  } catch (error) {
-    console.error('Error fetching files from knowledge document repository:', error);
-    return [];
-  }
-}
-
-// Fetch the latest commit info for a file
-async function fetchCommitInfo(headers: HeadersInit, owner: string, repo: string, filePath: string): Promise<{ sha: string; date: string } | null> {
-  try {
-    const response = await fetch(`${GITHUB_API_URL}/repos/${owner}/${repo}/commits?path=${filePath}`, { headers });
-    if (!response.ok) {
-      const errorText = await response.text();
-      console.error('Failed to fetch commit information for file:', response.status, errorText);
-      throw new Error('Failed to fetch commit information for file.');
-    }
-
-    const data = await response.json();
-    if (data.length === 0) return null;
-
-    return {
-      sha: data[0].sha,
-      date: data[0].commit.committer.date
-    };
-  } catch (error) {
-    console.error(`Error fetching commit info for ${filePath}:`, error);
-    return null;
-  }
-}
-
 // Fetch the content of a file from the repository
 async function fetchFileContent(headers: HeadersInit, owner: string, repo: string, filePath: string): Promise<string> {
   try {

src/app/api/github/pr/knowledge/route.ts

@@ -6,6 +6,7 @@ import { KnowledgeYamlData, AttributionData } from '@/types';
 import { GITHUB_API_URL, BASE_BRANCH } from '@/types/const';
 import { dumpYaml } from '@/utils/yamlConfig';
 import { checkUserForkExists, createBranch, createFilesInSingleCommit, createFork, getBaseBranchSha, getGitHubUsername } from '@/utils/github';
+import { prInfoFromSummary } from '@/app/api/github/utils';
 
 const KNOWLEDGE_DIR = 'knowledge';
 const UPSTREAM_REPO_OWNER = process.env.NEXT_PUBLIC_TAXONOMY_REPO_OWNER!;
@@ -29,7 +30,7 @@ export async function POST(req: NextRequest) {
 
   try {
     const body = await req.json();
-    const { content, attribution, name, email, submissionSummary, documentOutline, filePath } = body;
+    const { content, attribution, name, email, submissionSummary, filePath } = body;
 
     const knowledgeData: KnowledgeYamlData = yaml.load(content) as KnowledgeYamlData;
     const attributionData: AttributionData = attribution;
@@ -64,6 +65,8 @@ Creator names: ${attributionData.creator_names}
     // Create a new branch in the user's fork
     await createBranch(headers, githubUsername, UPSTREAM_REPO_NAME, branchName, baseBranchSha);
 
+    const { prTitle, prBody, commitMessage } = prInfoFromSummary(submissionSummary);
+
     // Create both files in a single commit with DCO sign-off
     await createFilesInSingleCommit(
       headers,
@@ -74,11 +77,11 @@ Creator names: ${attributionData.creator_names}
         { path: newAttributionFilePath, content: attributionContent }
       ],
       branchName,
-      `${submissionSummary}\n\nSigned-off-by: ${name} <${email}>`
+      `${commitMessage}\n\nSigned-off-by: ${name} <${email}>`
     );
 
     // Create a pull request from the user's fork to the upstream repository
-    const pr = await createPullRequest(headers, githubUsername, branchName, submissionSummary, documentOutline);
+    const pr = await createPullRequest(headers, githubUsername, branchName, prTitle, prBody);
 
     return NextResponse.json(pr, { status: 201 });
   } catch (error) {
@@ -87,14 +90,14 @@ Creator names: ${attributionData.creator_names}
   }
 }
 
-async function createPullRequest(headers: HeadersInit, username: string, branchName: string, knowledgeSummary: string, documentOutline: string) {
+async function createPullRequest(headers: HeadersInit, username: string, branchName: string, prTitle: string, prBody?: string) {
   const response = await fetch(`${GITHUB_API_URL}/repos/${UPSTREAM_REPO_OWNER}/${UPSTREAM_REPO_NAME}/pulls`, {
     method: 'POST',
     headers,
     body: JSON.stringify({
-      title: `Knowledge: ${knowledgeSummary}`,
+      title: `Knowledge: ${prTitle}`,
       head: `${username}:${branchName}`,
-      body: documentOutline,
+      body: prBody,
       base: BASE_BRANCH
     })
   });

src/app/api/github/pr/skill/route.ts

@@ -6,6 +6,7 @@ import { SkillYamlData, AttributionData } from '@/types';
 import { GITHUB_API_URL, BASE_BRANCH } from '@/types/const';
 import { dumpYaml } from '@/utils/yamlConfig';
 import { checkUserForkExists, createBranch, createFilesInSingleCommit, createFork, getBaseBranchSha, getGitHubUsername } from '@/utils/github';
+import { prInfoFromSummary } from '@/app/api/github/utils';
 
 const SKILLS_DIR = 'compositional_skills';
 const UPSTREAM_REPO_OWNER = process.env.NEXT_PUBLIC_TAXONOMY_REPO_OWNER!;
@@ -29,7 +30,7 @@ export async function POST(req: NextRequest) {
 
   try {
     const body = await req.json();
-    const { content, attribution, name, email, submissionSummary, documentOutline, filePath } = body;
+    const { content, attribution, name, email, submissionSummary, filePath } = body;
 
     const githubUsername = await getGitHubUsername(headers);
     console.log('Skill contribution from gitHub Username:', githubUsername);
@@ -62,6 +63,8 @@ Creator names: ${attributionData.creator_names}
     // Create a new branch in the user's fork
     await createBranch(headers, githubUsername, UPSTREAM_REPO_NAME, branchName, baseBranchSha);
 
+    const { prTitle, prBody, commitMessage } = prInfoFromSummary(submissionSummary);
+
     // Create both files in a single commit
     await createFilesInSingleCommit(
       headers,
@@ -72,11 +75,11 @@ Creator names: ${attributionData.creator_names}
         { path: newAttributionFilePath, content: attributionString }
       ],
       branchName,
-      `${submissionSummary}\n\nSigned-off-by: ${name} <${email}>`
+      `${commitMessage}\n\nSigned-off-by: ${name} <${email}>`
     );
 
     // Create a pull request from the user's fork to the upstream repository
-    const pr = await createPullRequest(headers, githubUsername, branchName, submissionSummary, documentOutline);
+    const pr = await createPullRequest(headers, githubUsername, branchName, prTitle, prBody);
 
     return NextResponse.json(pr, { status: 201 });
   } catch (error) {
@@ -85,14 +88,14 @@ Creator names: ${attributionData.creator_names}
   }
 }
 
-async function createPullRequest(headers: HeadersInit, username: string, branchName: string, skillSummary: string, skillDescription: string) {
+async function createPullRequest(headers: HeadersInit, username: string, branchName: string, prTitle: string, prBody?: string) {
   const response = await fetch(`${GITHUB_API_URL}/repos/${UPSTREAM_REPO_OWNER}/${UPSTREAM_REPO_NAME}/pulls`, {
     method: 'POST',
     headers,
     body: JSON.stringify({
-      title: `Skill: ${skillSummary}`,
+      title: `Skill: ${prTitle}`,
+      body: prBody,
       head: `${username}:${branchName}`,
-      body: skillDescription,
       base: BASE_BRANCH
     })
   });