- commit
- a50fc59
- parent
- 126a9cd
- author
- Eric Bower
- date
- 2021-08-01 13:30:54 +0000 UTC
improved scraping process
8 files changed,
+165,
-65
M
.nvmrc
+1,
-1
1@@ -1 +1 @@
2-12.21.0
3+14.17.4
+6,
-6
1@@ -11,9 +11,10 @@
2 "lint": "prettier --check --plugin-search-dir=. . && eslint --ignore-path .gitignore .",
3 "format": "prettier --write --plugin-search-dir=. .",
4 "prepare": "husky install",
5- "scrape": "node --experimental-specifier-resolution=node --loader ts-node/esm src/scrape.ts",
6- "process": "node --experimental-specifier-resolution=node --loader ts-node/esm src/process.ts",
7- "transform": "node --loader ts-node/esm src/transform.ts",
8+ "scrape": "node --experimental-specifier-resolution=node --loader ts-node/esm scripts/scrape.ts",
9+ "process": "node --experimental-specifier-resolution=node --loader ts-node/esm scripts/process.ts",
10+ "html": "node --loader ts-node/esm scripts/html.ts",
11+ "resource": "node --experimental-specifier-resolution=node --loader ts-node/esm scripts/resource.ts",
12 "upload:clean": "gsutil -m rm -r gs://neovimcraft.com/*",
13 "upload": "gsutil -m -h 'Cache-Control:private, max-age=0, no-transform' rsync -r ./build gs://neovimcraft.com",
14 "deploy": "yarn build:clean && yarn build && yarn upload:clean && yarn upload"
15@@ -24,6 +25,7 @@
16 "@types/marked": "^2.0.4",
17 "@types/node": "^16.3.3",
18 "@types/node-fetch": "^2.5.11",
19+ "@types/prettier": "^2.3.2",
20 "@typescript-eslint/eslint-plugin": "^4.19.0",
21 "@typescript-eslint/parser": "^4.19.0",
22 "eslint": "^7.22.0",
23@@ -49,7 +51,5 @@
24 "*.js": "eslint --cache --fix",
25 "*.{js,css,md}": "prettier --write"
26 },
27- "dependencies": {
28- "@types/prettier": "^2.3.2"
29- }
30+ "dependencies": {}
31 }
R src/transform.ts =>
scripts/html.ts
+1,
-1
1@@ -2,7 +2,7 @@ import fs from 'fs';
2 import util from 'util';
3 import marked from 'marked';
4
5-import type { Plugin } from './lib/types';
6+import type { Plugin } from '../src/lib/types';
7
8 const writeFile = util.promisify(fs.writeFile);
9 const readFile = util.promisify(fs.readFile);
R src/process.ts =>
scripts/process.ts
+45,
-12
1@@ -2,23 +2,53 @@ import fs from 'fs';
2 import util from 'util';
3 import fetch from 'node-fetch';
4
5-import type { Plugin, Resource } from './lib/types';
6-import { createPlugin } from './lib/entities';
7-import resourceFile from './lib/resources.json';
8+import type { Plugin, Resource } from '../src/lib/types';
9+import { createPlugin } from '../src/lib/entities';
10+import resourceFile from '../src/lib/resources.json';
11
12+const readFile = util.promisify(fs.readFile);
13 const writeFile = util.promisify(fs.writeFile);
14 const accessToken = process.env.GITHUB_ACCESS_TOKEN || '';
15 const accessUsername = process.env.GITHUB_USERNAME || '';
16
17-processResources(resourceFile.resources as Resource[])
18- .then(saveData)
19- .catch(console.error);
20+const args = process.argv;
21+const option = args[2];
22+if (option === 'missing') {
23+ console.log('PROCESSING MISSING RESOURCES');
24+ processMissingResources().then(saveData).catch(console.error);
25+} else {
26+ console.log('PROCESSING ALL RESOURCES');
27+ processResources(resourceFile.resources as Resource[])
28+ .then(saveData)
29+ .catch(console.error);
30+}
31
32 interface Props {
33 username: string;
34 repo: string;
35 }
36
37+async function processMissingResources() {
38+ const dbFile = await readFile('./src/lib/db.json');
39+ const db = JSON.parse(dbFile.toString());
40+ const missing: Resource[] = [];
41+ resourceFile.resources.forEach((r: Resource) => {
42+ if (db.plugins[`${r.username}/${r.repo}`]) {
43+ return;
44+ }
45+
46+ missing.push(r);
47+ });
48+ console.log(`Missing ${missing.length} resources`);
49+
50+ const results = await processResources(missing);
51+ const markdownFile = await readFile('./src/lib/markdown.json');
52+ const markdownJson = JSON.parse(markdownFile.toString());
53+ const plugins = { ...db.plugins, ...results.plugins };
54+ const markdown = { ...markdownJson.markdown, ...results.markdown };
55+ return { plugins, markdown };
56+}
57+
58 async function fetchReadme({
59 username,
60 repo,
61@@ -39,7 +69,6 @@ async function fetchReadme({
62 return { ok: true, data };
63 }
64
65- console.log(`FAILURE: could not load ${url}`);
66 return {
67 ok: false,
68 data: {
69@@ -64,7 +93,6 @@ async function fetchRepo({ username, repo }: Props): Promise<Resp<{ [key: string
70 };
71 }
72
73- console.log(`FAILURE: could not load ${url}`);
74 return {
75 ok: false,
76 data: {
77@@ -88,19 +116,25 @@ type Resp<D> = ApiSuccess<D> | ApiFailure;
78
79 async function fetchGithubData(props: Props): Promise<Resp<any>> {
80 const repo = await fetchRepo(props);
81- if (!repo.ok) return repo;
82+ if (repo.ok === false) {
83+ console.log(`${repo.data.status}: ${repo.data.error.message}`);
84+ return repo;
85+ }
86
87 const readme = await fetchReadme({
88 username: props.username,
89 repo: props.repo,
90 branch: repo.data.default_branch,
91 });
92- if (!readme.ok) return readme;
93+
94+ if (readme.ok === false) {
95+ console.log(`${readme.data.status}: ${readme.data.error.message}`);
96+ }
97
98 return {
99 ok: true,
100 data: {
101- readme: readme.data,
102+ readme: readme.ok ? readme.data : '',
103 repo: repo.data,
104 },
105 };
106@@ -149,7 +183,6 @@ async function saveData({
107 }: {
108 plugins: { [key: string]: Plugin };
109 markdown: { [key: string]: string };
110- resources: Resource[];
111 }) {
112 await writeFile('./src/lib/db.json', JSON.stringify({ plugins }));
113 await writeFile('./src/lib/markdown.json', JSON.stringify({ markdown }));
+59,
-0
1@@ -0,0 +1,59 @@
2+import fs from 'fs';
3+import util from 'util';
4+import readline from 'readline';
5+import prettier from 'prettier';
6+
7+import resourceFile from '../src/lib/resources.json';
8+import type { Resource } from '../src/lib/types';
9+import { createResource } from '../src/lib/entities';
10+
11+const writeFile = util.promisify(fs.writeFile);
12+
13+const rl = readline.createInterface({
14+ input: process.stdin,
15+ output: process.stdout,
16+});
17+
18+rl.on('close', function () {
19+ process.exit(0);
20+});
21+
22+const question = (inp: string) =>
23+ new Promise<string>((resolve) => {
24+ rl.question(inp, resolve);
25+ });
26+
27+async function init() {
28+ const name = await question('name (username/repo): ');
29+ const [username, repo] = name.split('/');
30+ const tagsRes = await question('tags (comma separated): ');
31+ const tags = tagsRes.split(',');
32+ const foundResource = (resourceFile.resources as Resource[]).find(
33+ (r) => `${r.username}/${r.repo}` === name,
34+ );
35+ if (foundResource) {
36+ console.log(`${name} aleady found in resources, not adding`);
37+ rl.close();
38+ return;
39+ }
40+
41+ resourceFile.resources.push(
42+ createResource({
43+ type: 'github',
44+ username,
45+ repo,
46+ tags,
47+ }),
48+ );
49+
50+ const json = prettier.format(JSON.stringify(resourceFile), {
51+ parser: 'json',
52+ printWidth: 100,
53+ });
54+
55+ await writeFile('./src/lib/resources.json', json);
56+
57+ rl.close();
58+}
59+
60+init().catch(console.error);
R src/scrape.ts =>
scripts/scrape.ts
+18,
-15
1@@ -5,9 +5,9 @@ import prettier from 'prettier';
2 import fetch from 'node-fetch';
3 import marked from 'marked';
4
5-import type { Resource, ResourceMap } from './lib/types';
6-import { createResource } from './lib/entities';
7-import resourceFile from './lib/resources.json';
8+import type { Resource, ResourceMap } from '../src/lib/types';
9+import { createResource } from '../src/lib/entities';
10+import resourceFile from '../src/lib/resources.json';
11
12 const writeFile = util.promisify(fs.writeFile);
13
14@@ -49,21 +49,24 @@ async function processMarkdown(text: string) {
15 token.items.forEach((t) => {
16 (t as any).tokens.forEach((tt: any) => {
17 if (!tt.tokens) return;
18- if (heading === 'contents') return;
19+
20+ // hardcoded deny-list for headings
21+ if (['contents', 'vim'].includes(heading)) return;
22 const resource = createResource({ tags: [sanitizeTag(heading)] });
23 let link = '';
24- tt.tokens.forEach((a: any) => {
25- if (a.type === 'link') {
26- link = a.href;
27- const href = a.href
28- .replace('https://github.com/', '')
29- .replace('http://github.com', '');
30- const d = href.split('/');
31- resource.username = d[0];
32- resource.repo = d[1];
33- }
34- });
35+
36+ // first token is always a link
37+ const token = tt.tokens[0];
38+ if (!token) return;
39+
40+ link = token.href;
41+ // skip non-github links
42 if (!link.includes('github.com')) return;
43+
44+ const href = link.replace('https://github.com/', '').replace('http://github.com', '');
45+ const d = href.split('/');
46+ resource.username = d[0];
47+ resource.repo = d[1];
48 resources.push(resource);
49 });
50 });
+13,
-8
1@@ -143,6 +143,12 @@
2 "repo": "surround.nvim",
3 "tags": ["editing-supports"]
4 },
5+ {
6+ "type": "github",
7+ "username": "LoricAndre",
8+ "repo": "OneTerm.nvim",
9+ "tags": ["terminal-integration"]
10+ },
11 { "type": "github", "username": "nikvdp", "repo": "neomux", "tags": ["terminal-integration"] },
12 {
13 "type": "github",
14@@ -209,10 +215,9 @@
15 "tags": ["fuzzy-finder"]
16 },
17 { "type": "github", "username": "vijaymarupudi", "repo": "nvim-fzf", "tags": ["fuzzy-finder"] },
18- { "type": "github", "username": "LoricAndre", "repo": "fzterm.nvim", "tags": ["fuzzy-finder"] },
19 { "type": "github", "username": "amirrezaask", "repo": "fuzzy.nvim", "tags": ["fuzzy-finder"] },
20 { "type": "github", "username": "camspiers", "repo": "snap", "tags": ["fuzzy-finder"] },
21- { "type": "github", "username": "lotabout", "repo": "skim", "tags": ["fuzzy-finder"] },
22+ { "type": "github", "username": "ibhagwan", "repo": "fzf-lua", "tags": ["fuzzy-finder"] },
23 { "type": "github", "username": "norcalli", "repo": "nvim-colorizer.lua", "tags": ["colors"] },
24 { "type": "github", "username": "tjdevries", "repo": "colorbuddy.nvim", "tags": ["colors"] },
25 { "type": "github", "username": "norcalli", "repo": "nvim-base16.lua", "tags": ["colors"] },
26@@ -500,8 +505,8 @@
27 { "type": "github", "username": "dracula", "repo": "vim", "tags": ["treesitter-colorschemes"] },
28 {
29 "type": "github",
30- "username": "saurabhdaware",
31- "repo": "vscode-calvera-dark",
32+ "username": "yashguptaz",
33+ "repo": "calvera-dark.nvim",
34 "tags": ["treesitter-colorschemes"]
35 },
36 {
37@@ -528,7 +533,7 @@
38 { "type": "github", "username": "clojure-vim", "repo": "jazz.nvim", "tags": ["utility"] },
39 { "type": "github", "username": "code-biscuits", "repo": "nvim-biscuits", "tags": ["utility"] },
40 { "type": "github", "username": "Pocco81", "repo": "AbbrevMan.nvim", "tags": ["utility"] },
41- { "type": "github", "username": "ryanoasis", "repo": "vim-devicons", "tags": ["icons"] },
42+ { "type": "github", "username": "kyazdani42", "repo": "nvim-web-devicons", "tags": ["icons"] },
43 { "type": "github", "username": "yamatsum", "repo": "nvim-nonicons", "tags": ["icons"] },
44 { "type": "github", "username": "mfussenegger", "repo": "nvim-dap", "tags": ["debugging"] },
45 { "type": "github", "username": "sakhnik", "repo": "nvim-gdb", "tags": ["debugging"] },
46@@ -627,7 +632,7 @@
47 },
48 { "type": "github", "username": "hoob3rt", "repo": "lualine.nvim", "tags": ["statusline"] },
49 { "type": "github", "username": "adelarsq", "repo": "neoline.vim", "tags": ["statusline"] },
50- { "type": "github", "username": "vim-airline", "repo": "vim-airline", "tags": ["statusline"] },
51+ { "type": "github", "username": "ojroques", "repo": "nvim-hardline", "tags": ["statusline"] },
52 { "type": "github", "username": "datwaft", "repo": "bubbly.nvim", "tags": ["statusline"] },
53 {
54 "type": "github",
55@@ -926,8 +931,8 @@
56 { "type": "github", "username": "NTBBloodbath", "repo": "nvenv", "tags": ["version-managers"] },
57 { "type": "github", "username": "shohi", "repo": "neva", "tags": ["version-managers"] },
58 { "type": "github", "username": "gennaro-tedesco", "repo": "boilit", "tags": ["boilerplate"] },
59- { "type": "github", "username": "akrawchyk", "repo": "awesome-vim#tools", "tags": ["vim"] },
60 { "type": "github", "username": "tsbohc", "repo": "zest.nvim", "tags": ["fennel"] },
61- { "type": "github", "username": "fhill2", "repo": "floating.nvim", "tags": ["utility"] }
62+ { "type": "github", "username": "fhill2", "repo": "floating.nvim", "tags": ["utility"] },
63+ { "type": "github", "username": "caenrique", "repo": "swap-buffers.nvim", "tags": ["buffers"] }
64 ]
65 }
+22,
-22
1@@ -1,30 +1,30 @@
2 {
3- "compilerOptions": {
4- "moduleResolution": "node",
5- "module": "es2020",
6- "lib": ["es2020"],
7- "target": "es2019",
8- /**
9+ "compilerOptions": {
10+ "moduleResolution": "node",
11+ "module": "es2020",
12+ "lib": ["es2020"],
13+ "target": "es2019",
14+ /**
15 svelte-preprocess cannot figure out whether you have a value or a type, so tell TypeScript
16 to enforce using \`import type\` instead of \`import\` for Types.
17 */
18- "importsNotUsedAsValues": "error",
19- "isolatedModules": true,
20- "resolveJsonModule": true,
21- /**
22+ "importsNotUsedAsValues": "error",
23+ "isolatedModules": true,
24+ "resolveJsonModule": true,
25+ /**
26 To have warnings/errors of the Svelte compiler at the correct position,
27 enable source maps by default.
28 */
29- "sourceMap": true,
30- "esModuleInterop": true,
31- "skipLibCheck": true,
32- "forceConsistentCasingInFileNames": true,
33- "baseUrl": ".",
34- "allowJs": true,
35- "checkJs": true,
36- "paths": {
37- "$lib/*": ["src/lib/*"]
38- }
39- },
40- "include": ["src/**/*.d.ts", "src/**/*.js", "src/**/*.ts", "src/**/*.svelte"]
41+ "sourceMap": true,
42+ "esModuleInterop": true,
43+ "skipLibCheck": true,
44+ "forceConsistentCasingInFileNames": true,
45+ "baseUrl": ".",
46+ "allowJs": true,
47+ "checkJs": true,
48+ "paths": {
49+ "$lib/*": ["src/lib/*"]
50+ }
51+ },
52+ "include": ["src/**/*.d.ts", "src/**/*.js", "src/**/*.ts", "src/**/*.svelte", "scripts/*.ts"]
53 }