repos / neovimcraft

scrape-config.ts

 1import { fetchTopics, ghToken } from "../github.ts";
 2import { createResource, getResourceId } from "../entities.ts";
 3import { ResourceMap } from "../types.ts";
 4import { denyRepos } from "../filter.ts";
 5
 6init().catch(console.error);
 7
 8async function init() {
 9  const topics = [
10    "neovim-config",
11    "neovim-configs",
12    "neovim-dotfiles",
13    "neovim-setup",
14    "neovim-configuration",
15  ];
16  const resources: ResourceMap = {};
17  for (const topic of topics) {
18    await dl(resources, topic);
19  }
20  console.log(`Found ${Object.values(resources).length} repos`);
21  await save(resources);
22}
23
24async function save(resources: ResourceMap) {
25  const data = { resources: Object.values(resources) };
26  const json = JSON.stringify(data, null, 2);
27
28  await Deno.writeTextFile("./data/scrape-config.json", json);
29}
30
31async function dl(resources: ResourceMap, topic: string) {
32  const repos = await fetchTopics(topic, ghToken);
33  console.log(`${topic} found ${repos.length} repos`);
34  const resourceList = repos
35    .map((repo) => {
36      const [username, repoName] = repo.full_name.split("/");
37      return createResource({
38        username,
39        repo: repoName,
40        tags: repo.topics,
41      });
42    }).filter((repo) => !denyRepos.includes(getResourceId(repo)));
43  const newResources = resourceList.reduce<ResourceMap>((acc, repo) => {
44    acc[getResourceId(repo)] = repo;
45    return acc;
46  }, resources);
47
48  return newResources;
49}