repos / neovimcraft

github.ts

  1import type { FetchRepoProps, Resp } from "./types.ts";
  2
  3import { encode } from "./deps.ts";
  4
  5const accessToken = Deno.env.get("GITHUB_ACCESS_TOKEN") || "";
  6const accessUsername = Deno.env.get("GITHUB_USERNAME") || "";
  7export const ghToken = encode(`${accessUsername}:${accessToken}`);
  8
  9function delay(ms: number): Promise<void> {
 10  return new Promise((resolve) => setTimeout(() => resolve(), ms));
 11}
 12
 13const nextRe = new RegExp(/\<([^<]+)\>; rel="next"/);
 14
 15async function githubApi<D = any>(
 16  endpoint: string,
 17  token: string,
 18): Promise<Resp<D>> {
 19  const url = `https://api.github.com${endpoint}`;
 20  console.log(`Fetching ${url}`);
 21  const res = await fetch(url, {
 22    headers: { Authorization: `Basic ${token}` },
 23  });
 24
 25  const rateLimitRemaining = parseInt(
 26    res.headers.get("X-RateLimit-Remaining") || "0",
 27  );
 28  // this value is in seconds, not ms
 29  const rateLimitReset = parseInt(res.headers.get("X-RateLimit-Reset") || "0");
 30  console.log(`rate limit remaining: ${rateLimitRemaining}`);
 31  if (rateLimitRemaining === 1 || rateLimitRemaining === 0) {
 32    const now = Date.now();
 33    const RESET_BUFFER = 500;
 34    const wait = (rateLimitReset * 1000) + RESET_BUFFER - now;
 35    // ms -> s -> min
 36    const mins = Math.ceil(wait / 1000 / 60);
 37    console.log(
 38      `Hit github rate limit, waiting ${mins} mins`,
 39    );
 40    await delay(wait);
 41  }
 42
 43  let data = null;
 44  try {
 45    data = await res.json();
 46  } catch {
 47    return {
 48      ok: false,
 49      data: {
 50        status: res.status,
 51        error: new Error(`JSON parsing error [${url}]`),
 52      },
 53    };
 54  }
 55
 56  if (res.ok) {
 57    // pagination
 58    let next = "";
 59    const link = res.headers.get("link");
 60    if (link) {
 61      const paginated = nextRe.exec(link || "");
 62      if (paginated && paginated.length > 1) {
 63        next = paginated[1];
 64      }
 65    }
 66
 67    return {
 68      ok: true,
 69      next,
 70      data,
 71    };
 72  }
 73
 74  return {
 75    ok: false,
 76    data: {
 77      status: res.status,
 78      error: new Error(`Could not load [${url}]`),
 79    },
 80  };
 81}
 82
 83async function fetchReadme({
 84  username,
 85  repo,
 86  token,
 87}: FetchRepoProps): Promise<Resp<string>> {
 88  const result = await githubApi(`/repos/${username}/${repo}/readme`, token);
 89  if (!result.ok) {
 90    return {
 91      ok: false,
 92      data: result.data as any,
 93    };
 94  }
 95
 96  const url = result.data.download_url;
 97  console.log(`Fetching ${url}`);
 98  const readme = await fetch(url);
 99  const data = await readme.text();
100  return {
101    ok: true,
102    data,
103  };
104}
105
106interface RepoData {
107  name: string;
108  html_url: string;
109  homepage: string;
110  default_branch: string;
111  open_issues_count: number;
112  watchers_count: number;
113  forks: number;
114  stargazers_count: number;
115  subscribers_count: number;
116  network_count: number;
117  description: string;
118  created_at: string;
119  updated_at: string;
120}
121
122async function fetchRepo({
123  username,
124  repo,
125  token,
126}: FetchRepoProps): Promise<Resp<RepoData>> {
127  const result = await githubApi(`/repos/${username}/${repo}`, token);
128  return result;
129}
130
131interface BranchData {
132  commit: {
133    commit: {
134      committer: {
135        date: string;
136      };
137    };
138  };
139}
140
141async function fetchBranch({
142  username,
143  repo,
144  branch,
145  token,
146}: FetchRepoProps & { branch: string }): Promise<Resp<BranchData>> {
147  const result = await githubApi(
148    `/repos/${username}/${repo}/branches/${branch}`,
149    token,
150  );
151  return result;
152}
153
154interface GithubData {
155  readme: string;
156  repo: RepoData;
157  branch: Resp<BranchData>;
158}
159
160export async function fetchGithubData(
161  props: FetchRepoProps,
162): Promise<Resp<GithubData>> {
163  const repo = await fetchRepo(props);
164  if (repo.ok === false) {
165    console.log(`${repo.data.status}: ${repo.data.error.message}`);
166    return repo;
167  }
168
169  const branch = await fetchBranch({
170    ...props,
171    branch: repo.data.default_branch,
172  });
173  if (branch.ok === false) {
174    console.log(`${branch.data.status}: ${branch.data.error.message}`);
175  }
176
177  const readme = await fetchReadme({
178    username: props.username,
179    repo: props.repo,
180    token: props.token,
181  });
182  if (readme.ok === false) {
183    console.log(`${readme.data.status}: ${readme.data.error.message}`);
184  }
185
186  return {
187    ok: true,
188    data: {
189      readme: readme.ok ? readme.data : "",
190      repo: repo.data,
191      branch,
192    },
193  };
194}
195
196type TopicRepo = RepoData & { full_name: string; topics: string[] };
197
198export async function fetchTopics(topic: string, token: string) {
199  const items: TopicRepo[] = [];
200  let next = `/search/repositories?q=${topic}&per_page=25`;
201  // limit the number of pages
202  // let count = 0;
203
204  while (next) {
205    const result = await githubApi<{ items: TopicRepo[] }>(
206      next,
207      token,
208    );
209
210    next = "";
211
212    if (result.ok) {
213      items.push(...result.data.items);
214
215      /* count += 1;
216      if (result.next && count < 3) {
217        next = result.next;
218        next = next.replace('https://api.github.com', '');
219      } */
220    }
221  }
222
223  return items;
224}