Skip to content

Commit 92e3deb

Browse files
tom2drumcarlomigueldy
tom2drum
authored and
carlomigueldy
committed
Generate sitemap.xml and robots.txt files (blockscout#2502)
* base implementation of sitemap generator * move generator to separate directory to optimize docker image size * generate robots.txt file
1 parent 0cc335c commit 92e3deb

File tree

10 files changed

+373
-2
lines changed

10 files changed

+373
-2
lines changed

‎.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
/public/icons/sprite.svg
1818
/public/icons/sprite.*.svg
1919
/public/icons/README.md
20+
/public/sitemap.xml
21+
/public/robots.txt
2022
/analyze
2123

2224
# production

‎Dockerfile

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,12 @@ WORKDIR /favicon-generator
3333
COPY ./deploy/tools/favicon-generator/package.json ./deploy/tools/favicon-generator/yarn.lock ./
3434
RUN yarn --frozen-lockfile --network-timeout 100000
3535

36+
### SITEMAP GENERATOR
37+
# Install dependencies
38+
WORKDIR /sitemap-generator
39+
COPY ./deploy/tools/sitemap-generator/package.json ./deploy/tools/sitemap-generator/yarn.lock ./
40+
RUN yarn --frozen-lockfile --network-timeout 100000
41+
3642

3743
# *****************************
3844
# ****** STAGE 2: Build *******
@@ -88,6 +94,12 @@ RUN cd ./deploy/tools/envs-validator && yarn build
8894
# Copy dependencies and source code
8995
COPY --from=deps /favicon-generator/node_modules ./deploy/tools/favicon-generator/node_modules
9096

97+
98+
### SITEMAP GENERATOR
99+
# Copy dependencies and source code
100+
COPY --from=deps /sitemap-generator/node_modules ./deploy/tools/sitemap-generator/node_modules
101+
102+
91103
# *****************************
92104
# ******* STAGE 3: Run ********
93105
# *****************************
@@ -127,6 +139,9 @@ COPY --chmod=755 ./deploy/scripts/favicon_generator.sh .
127139
COPY --from=builder /app/deploy/tools/favicon-generator ./deploy/tools/favicon-generator
128140
RUN ["chmod", "-R", "777", "./deploy/tools/favicon-generator"]
129141
RUN ["chmod", "-R", "777", "./public"]
142+
## Sitemap generator
143+
COPY --chmod=755 ./deploy/scripts/sitemap_generator.sh .
144+
COPY --from=builder /app/deploy/tools/sitemap-generator ./deploy/tools/sitemap-generator
130145

131146
# Copy ENVs files
132147
COPY --from=builder /app/.env.registry .

‎deploy/scripts/entrypoint.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,9 @@ echo
6161
# Create envs.js file with run-time environment variables for the client app
6262
./make_envs_script.sh
6363

64+
# Generate sitemap.xml and robots.txt files
65+
./sitemap_generator.sh
66+
6467
# Print list of enabled features
6568
node ./feature-reporter.js
6669

‎deploy/scripts/sitemap_generator.sh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
cd ./deploy/tools/sitemap-generator
2+
yarn next-sitemap

‎deploy/tools/sitemap-generator/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
/node_modules

‎deploy/tools/sitemap-generator/next-sitemap.config.js

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
/* eslint-disable no-console */
2+
const path = require('path');
3+
4+
const stripTrailingSlash = (str) => str[str.length - 1] === '/' ? str.slice(0, -1) : str;
5+
6+
const fetchResource = async(url, formatter) => {
7+
console.log('🌀 [next-sitemap] Fetching resource:', url);
8+
try {
9+
const res = await fetch(url);
10+
if (res.ok) {
11+
const data = await res.json();
12+
console.log('✅ [next-sitemap] Data fetched for resource:', url);
13+
return formatter(data);
14+
}
15+
} catch (error) {
16+
console.log('🚨 [next-sitemap] Error fetching resource:', url, error);
17+
}
18+
};
19+
20+
const siteUrl = [
21+
process.env.NEXT_PUBLIC_APP_PROTOCOL || 'https',
22+
'://',
23+
process.env.NEXT_PUBLIC_APP_HOST,
24+
process.env.NEXT_PUBLIC_APP_PORT && ':' + process.env.NEXT_PUBLIC_APP_PORT,
25+
].filter(Boolean).join('');
26+
27+
const apiUrl = (() => {
28+
const baseUrl = [
29+
process.env.NEXT_PUBLIC_API_PROTOCOL || 'https',
30+
'://',
31+
process.env.NEXT_PUBLIC_API_HOST,
32+
process.env.NEXT_PUBLIC_API_PORT && ':' + process.env.NEXT_PUBLIC_API_PORT,
33+
].filter(Boolean).join('');
34+
35+
const basePath = stripTrailingSlash(process.env.NEXT_PUBLIC_API_BASE_PATH || '');
36+
37+
return `${ baseUrl }${ basePath }/api/v2`;
38+
})();
39+
40+
/** @type {import('next-sitemap').IConfig} */
41+
module.exports = {
42+
siteUrl,
43+
generateIndexSitemap: false,
44+
generateRobotsTxt: true,
45+
sourceDir: path.resolve(process.cwd(), '../../../.next'),
46+
outDir: path.resolve(process.cwd(), '../../../public'),
47+
exclude: [
48+
'/account/*',
49+
'/auth/*',
50+
'/login',
51+
'/sprite',
52+
],
53+
transform: async(config, path) => {
54+
switch (path) {
55+
case '/mud-worlds':
56+
if (process.env.NEXT_PUBLIC_HAS_MUD_FRAMEWORK !== 'true') {
57+
return null;
58+
}
59+
break;
60+
case '/batches':
61+
case '/deposits':
62+
if (!process.env.NEXT_PUBLIC_ROLLUP_TYPE) {
63+
return null;
64+
}
65+
break;
66+
case '/withdrawals':
67+
if (!process.env.NEXT_PUBLIC_ROLLUP_TYPE && process.env.NEXT_PUBLIC_HAS_BEACON_CHAIN !== 'true') {
68+
return null;
69+
}
70+
break;
71+
case '/dispute-games':
72+
if (process.env.NEXT_PUBLIC_ROLLUP_TYPE !== 'optimistic') {
73+
return null;
74+
}
75+
break;
76+
case '/blobs':
77+
if (process.env.NEXT_PUBLIC_DATA_AVAILABILITY_ENABLED !== 'true') {
78+
return null;
79+
}
80+
break;
81+
case '/name-domains':
82+
if (!process.env.NEXT_PUBLIC_NAME_SERVICE_API_HOST) {
83+
return null;
84+
}
85+
break;
86+
case '/ops':
87+
if (process.env.NEXT_PUBLIC_HAS_USER_OPS !== 'true') {
88+
return null;
89+
}
90+
break;
91+
case '/output-roots':
92+
if (process.env.NEXT_PUBLIC_ROLLUP_OUTPUT_ROOTS_ENABLED !== 'true') {
93+
return null;
94+
}
95+
break;
96+
case '/pools':
97+
if (process.env.NEXT_PUBLIC_DEX_POOLS_ENABLED !== 'true') {
98+
return null;
99+
}
100+
break;
101+
case '/advanced-filter':
102+
if (process.env.NEXT_PUBLIC_ADVANCED_FILTER_ENABLED === 'false') {
103+
return null;
104+
}
105+
break;
106+
case '/apps':
107+
if (process.env.NEXT_PUBLIC_MARKETPLACE_ENABLED !== 'true') {
108+
return null;
109+
}
110+
break;
111+
case '/api-docs':
112+
if (process.env.NEXT_PUBLIC_API_SPEC_URL === 'none') {
113+
return null;
114+
}
115+
break;
116+
case '/gas-tracker':
117+
if (process.env.NEXT_PUBLIC_GAS_TRACKER_ENABLED === 'false') {
118+
return null;
119+
}
120+
break;
121+
case '/graphql':
122+
if (process.env.NEXT_PUBLIC_GRAPHIQL_TRANSACTION === 'none') {
123+
return null;
124+
}
125+
break;
126+
case '/stats':
127+
if (!process.env.NEXT_PUBLIC_STATS_API_HOST) {
128+
return null;
129+
}
130+
break;
131+
case '/validators':
132+
if (!process.env.NEXT_PUBLIC_VALIDATORS_CHAIN_TYPE) {
133+
return null;
134+
}
135+
break;
136+
}
137+
138+
return {
139+
loc: path,
140+
changefreq: undefined,
141+
priority: undefined,
142+
lastmod: config.autoLastmod ? new Date().toISOString() : undefined,
143+
alternateRefs: config.alternateRefs ?? [],
144+
};
145+
},
146+
additionalPaths: async(config) => {
147+
const addresses = fetchResource(
148+
`${ apiUrl }/addresses`,
149+
(data) => data.items.map(({ hash }) => `/address/${ hash }`),
150+
);
151+
const txs = fetchResource(
152+
`${ apiUrl }/transactions?filter=validated`,
153+
(data) => data.items.map(({ hash }) => `/tx/${ hash }`),
154+
);
155+
const blocks = fetchResource(
156+
`${ apiUrl }/blocks?type=block`,
157+
(data) => data.items.map(({ height }) => `/block/${ height }`),
158+
);
159+
const tokens = fetchResource(
160+
`${ apiUrl }/tokens`,
161+
(data) => data.items.map(({ address }) => `/token/${ address }`),
162+
);
163+
const contracts = fetchResource(
164+
`${ apiUrl }/smart-contracts`,
165+
(data) => data.items.map(({ address }) => `/address/${ address.hash }?tab=contract`),
166+
);
167+
168+
return Promise.all([
169+
...(await addresses || []),
170+
...(await txs || []),
171+
...(await blocks || []),
172+
...(await tokens || []),
173+
...(await contracts || []),
174+
].map(path => config.transform(config, path)));
175+
},
176+
};

‎deploy/tools/sitemap-generator/package.json

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"name": "sitemap-generator",
3+
"version": "1.0.0",
4+
"main": "index.js",
5+
"keywords": [],
6+
"author": "",
7+
"license": "ISC",
8+
"description": "",
9+
"dependencies": {
10+
"next-sitemap": "4.2.3"
11+
}
12+
}

‎deploy/tools/sitemap-generator/yarn.lock

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
2+
# yarn lockfile v1
3+
4+
5+
"@corex/deepmerge@^4.0.43":
6+
version "4.0.43"
7+
resolved "https://registry.yarnpkg.com/@corex/deepmerge/-/deepmerge-4.0.43.tgz#9bd42559ebb41cc5a7fb7cfeea5f231c20977dca"
8+
integrity sha512-N8uEMrMPL0cu/bdboEWpQYb/0i2K5Qn8eCsxzOmxSggJbbQte7ljMRoXm917AbntqTGOzdTu+vP3KOOzoC70HQ==
9+
10+
"@next/env@^13.4.3":
11+
version "13.5.8"
12+
resolved "https://registry.yarnpkg.com/@next/env/-/env-13.5.8.tgz#404d3b3e5881b6a0510500c6cc97e3589a2e6371"
13+
integrity sha512-YmiG58BqyZ2FjrF2+5uZExL2BrLr8RTQzLXNDJ8pJr0O+rPlOeDPXp1p1/4OrR3avDidzZo3D8QO2cuDv1KCkw==
14+
15+
"@nodelib/[email protected]":
16+
version "2.1.5"
17+
resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5"
18+
integrity sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==
19+
dependencies:
20+
"@nodelib/fs.stat" "2.0.5"
21+
run-parallel "^1.1.9"
22+
23+
"@nodelib/[email protected]", "@nodelib/fs.stat@^2.0.2":
24+
version "2.0.5"
25+
resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz#5bd262af94e9d25bd1e71b05deed44876a222e8b"
26+
integrity sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==
27+
28+
"@nodelib/fs.walk@^1.2.3":
29+
version "1.2.8"
30+
resolved "https://registry.yarnpkg.com/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz#e95737e8bb6746ddedf69c556953494f196fe69a"
31+
integrity sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==
32+
dependencies:
33+
"@nodelib/fs.scandir" "2.1.5"
34+
fastq "^1.6.0"
35+
36+
braces@^3.0.3:
37+
version "3.0.3"
38+
resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789"
39+
integrity sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==
40+
dependencies:
41+
fill-range "^7.1.1"
42+
43+
fast-glob@^3.2.12:
44+
version "3.3.2"
45+
resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.3.2.tgz#a904501e57cfdd2ffcded45e99a54fef55e46129"
46+
integrity sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==
47+
dependencies:
48+
"@nodelib/fs.stat" "^2.0.2"
49+
"@nodelib/fs.walk" "^1.2.3"
50+
glob-parent "^5.1.2"
51+
merge2 "^1.3.0"
52+
micromatch "^4.0.4"
53+
54+
fastq@^1.6.0:
55+
version "1.18.0"
56+
resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.18.0.tgz#d631d7e25faffea81887fe5ea8c9010e1b36fee0"
57+
integrity sha512-QKHXPW0hD8g4UET03SdOdunzSouc9N4AuHdsX8XNcTsuz+yYFILVNIX4l9yHABMhiEI9Db0JTTIpu0wB+Y1QQw==
58+
dependencies:
59+
reusify "^1.0.4"
60+
61+
fill-range@^7.1.1:
62+
version "7.1.1"
63+
resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.1.1.tgz#44265d3cac07e3ea7dc247516380643754a05292"
64+
integrity sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==
65+
dependencies:
66+
to-regex-range "^5.0.1"
67+
68+
glob-parent@^5.1.2:
69+
version "5.1.2"
70+
resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4"
71+
integrity sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==
72+
dependencies:
73+
is-glob "^4.0.1"
74+
75+
is-extglob@^2.1.1:
76+
version "2.1.1"
77+
resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2"
78+
integrity sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==
79+
80+
is-glob@^4.0.1:
81+
version "4.0.3"
82+
resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.3.tgz#64f61e42cbbb2eec2071a9dac0b28ba1e65d5084"
83+
integrity sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==
84+
dependencies:
85+
is-extglob "^2.1.1"
86+
87+
is-number@^7.0.0:
88+
version "7.0.0"
89+
resolved "https://registry.yarnpkg.com/is-number/-/is-number-7.0.0.tgz#7535345b896734d5f80c4d06c50955527a14f12b"
90+
integrity sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==
91+
92+
merge2@^1.3.0:
93+
version "1.4.1"
94+
resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae"
95+
integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==
96+
97+
micromatch@^4.0.4:
98+
version "4.0.8"
99+
resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.8.tgz#d66fa18f3a47076789320b9b1af32bd86d9fa202"
100+
integrity sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==
101+
dependencies:
102+
braces "^3.0.3"
103+
picomatch "^2.3.1"
104+
105+
minimist@^1.2.8:
106+
version "1.2.8"
107+
resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c"
108+
integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==
109+
110+
[email protected]:
111+
version "4.2.3"
112+
resolved "https://registry.yarnpkg.com/next-sitemap/-/next-sitemap-4.2.3.tgz#5db3f650351a51e84b9fd6b58c5af2f9257b5058"
113+
integrity sha512-vjdCxeDuWDzldhCnyFCQipw5bfpl4HmZA7uoo3GAaYGjGgfL4Cxb1CiztPuWGmS+auYs7/8OekRS8C2cjdAsjQ==
114+
dependencies:
115+
"@corex/deepmerge" "^4.0.43"
116+
"@next/env" "^13.4.3"
117+
fast-glob "^3.2.12"
118+
minimist "^1.2.8"
119+
120+
picomatch@^2.3.1:
121+
version "2.3.1"
122+
resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42"
123+
integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==
124+
125+
queue-microtask@^1.2.2:
126+
version "1.2.3"
127+
resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243"
128+
integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==
129+
130+
reusify@^1.0.4:
131+
version "1.0.4"
132+
resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76"
133+
integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==
134+
135+
run-parallel@^1.1.9:
136+
version "1.2.0"
137+
resolved "https://registry.yarnpkg.com/run-parallel/-/run-parallel-1.2.0.tgz#66d1368da7bdf921eb9d95bd1a9229e7f21a43ee"
138+
integrity sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==
139+
dependencies:
140+
queue-microtask "^1.2.2"
141+
142+
to-regex-range@^5.0.1:
143+
version "5.0.1"
144+
resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-5.0.1.tgz#1648c44aae7c8d988a326018ed72f5b4dd0392e4"
145+
integrity sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==
146+
dependencies:
147+
is-number "^7.0.0"

0 commit comments

Comments
 (0)