Commit e78b314a authored by Emmanuel Raviart's avatar Emmanuel Raviart
Browse files

Add scripts once again and make them work.

parent 4a488043
......@@ -4,6 +4,7 @@ node_modules
!/example.env
/.svelte-kit
/build/
/dist/
/package/
static/img_old/
data_catalogue_2021-02-07.sql
......
......@@ -47,6 +47,12 @@ As normal user, install dependencies:
npm install
```
As normal user, compile scripts to be able to use them:
```bash
npm run build-scripts
```
As normal user, create database tables:
```bash
......@@ -61,40 +67,40 @@ npm run configure
```bash
# ADISP (OAI-PMH): Contains every french DDIs
npx babel-node --extensions ".ts" -- src/scripts/retrieve_oai-pmh_ddis.ts --url http://www.progedo-adisp.fr/oai/oai2.php ../public_data/adisp-oai-pmh-ddi/
node --experimental-specifier-resolution=node -- dist/scripts/retrieve_oai-pmh_ddis.js --url http://www.progedo-adisp.fr/oai/oai2.php ../public_data/adisp-oai-pmh-ddi/
```
#### Fetching DDI Files from Dataverse Servers
```bash
# data.sciencespo
npx babel-node --extensions ".ts" -- src/scripts/retrieve_dataverse_ddis.ts --url https://data.sciencespo.fr/ --verbose ../public_data/sciencespo-dataverse-ddi/
node --experimental-specifier-resolution=node -- dist/scripts/retrieve_dataverse_ddis.js --url https://data.sciencespo.fr/ --verbose ../public_data/sciencespo-dataverse-ddi/
```
#### Fetching DDI Files from Nesstar Servers
```bash
# ADISP (public Nesstar) : Contains some French & English DDIs
npx babel-node --extensions ".ts" src/scripts/retrieve_nesstar_ddis.ts --url http://nesstar.progedo-adisp.fr/ ../public_data/adisp-nesstar-ddi/
node --experimental-specifier-resolution=node dist/scripts/retrieve_nesstar_ddis.js --url http://nesstar.progedo-adisp.fr/ ../public_data/adisp-nesstar-ddi/
# CDSP Sciences Po (obsolete & closed)
# npx babel-node --extensions ".ts" src/scripts/retrieve_nesstar_ddis.ts --url http://nesstar.sciences-po.fr/ ../public_data/cdsp-nesstar-ddi/
# node --experimental-specifier-resolution=node dist/scripts/retrieve_nesstar_ddis.js --url http://nesstar.sciences-po.fr/ ../public_data/cdsp-nesstar-ddi/
# INED
npx babel-node --extensions ".ts" src/scripts/retrieve_nesstar_ddis.ts --url http://nesstar.ined.fr/ ../public_data/ined-nesstar-ddi/
node --experimental-specifier-resolution=node dist/scripts/retrieve_nesstar_ddis.js --url http://nesstar.ined.fr/ ../public_data/ined-nesstar-ddi/
# INED - Generations and Gender Survey
npx babel-node --extensions ".ts" src/scripts/retrieve_nesstar_ddis.ts --url http://ggpsurvey.ined.fr/ ../public_data/ined-gpgsurvey-nesstar-ddi/
node --experimental-specifier-resolution=node dist/scripts/retrieve_nesstar_ddis.js --url http://ggpsurvey.ined.fr/ ../public_data/ined-gpgsurvey-nesstar-ddi/
# UK Data Service
npx babel-node --extensions ".ts" src/scripts/retrieve_nesstar_ddis.ts --url http://nesstar.ukdataservice.ac.uk/ ../public_data/ukdataservice-nesstar-ddi/
node --experimental-specifier-resolution=node dist/scripts/retrieve_nesstar_ddis.js --url http://nesstar.ukdataservice.ac.uk/ ../public_data/ukdataservice-nesstar-ddi/
# Norwegian Centre for Research Data
npx babel-node --extensions ".ts" src/scripts/retrieve_nesstar_ddis.ts --url http://nsddata.nsd.uib.no ../public_data/nsddata-nesstar-ddi/
node --experimental-specifier-resolution=node dist/scripts/retrieve_nesstar_ddis.js --url http://nsddata.nsd.uib.no ../public_data/nsddata-nesstar-ddi/
```
### Repairing DDI Files
```bash
# ADISP
npx babel-node --extensions ".ts" src/scripts/repair_adisp_oai-pmh_ddis.ts --source=../public_data/adisp-oai-pmh-ddi/ ../public_data/adisp-oai-pmh-ddi-repaired/
npx babel-node --extensions ".ts" src/scripts/repair_adisp_nesstar_ddis.ts --source=../public_data/adisp-nesstar-ddi/ ../public_data/adisp-nesstar-ddi-repaired/
npx babel-node --extensions ".ts" src/scripts/repair_ined_nesstar_ddis.ts --source=../public_data/ined-nesstar-ddi/ ../public_data/ined-nesstar-ddi-repaired/
node --experimental-specifier-resolution=node dist/scripts/repair_adisp_oai-pmh_ddis.js --source=../public_data/adisp-oai-pmh-ddi/ ../public_data/adisp-oai-pmh-ddi-repaired/
node --experimental-specifier-resolution=node dist/scripts/repair_adisp_nesstar_ddis.js --source=../public_data/adisp-nesstar-ddi/ ../public_data/adisp-nesstar-ddi-repaired/
node --experimental-specifier-resolution=node dist/scripts/repair_ined_nesstar_ddis.js --source=../public_data/ined-nesstar-ddi/ ../public_data/ined-nesstar-ddi-repaired/
```
### Indexing DDI files
......@@ -102,28 +108,28 @@ npx babel-node --extensions ".ts" src/scripts/repair_ined_nesstar_ddis.ts --sour
#### Indexing Progedo DDI Files
```bash
npx babel-node --extensions ".ts" -- src/scripts/index_codebooks.ts --language=fr --path=adisp ../public_data/adisp-oai-pmh-ddi-repaired/
npx babel-node --extensions ".ts" -- src/scripts/index_codebooks.ts --language=fr --path=cdsp ../public_data/sciencespo-dataverse-ddi/
npx babel-node --extensions ".ts" -- src/scripts/index_codebooks.ts --language=fr --path=ined ../public_data/ined-nesstar-ddi-repaired/
node --experimental-specifier-resolution=node -- dist/scripts/index_codebooks.js --language=fr --path=adisp ../public_data/adisp-oai-pmh-ddi-repaired/
node --experimental-specifier-resolution=node -- dist/scripts/index_codebooks.js --language=fr --path=cdsp ../public_data/sciencespo-dataverse-ddi/
node --experimental-specifier-resolution=node -- dist/scripts/index_codebooks.js --language=fr --path=ined ../public_data/ined-nesstar-ddi-repaired/
```
#### Indexing Other (non Progedo-related) DDI Files
```bash
npx babel-node --extensions ".ts" -- src/scripts/index_codebooks.ts --language=fr --path=adisp-nesstar ../public_data/adisp-nesstar-ddi-repaired/
npx babel-node --extensions ".ts" -- src/scripts/index_codebooks.ts --language=fr --path=cdsp ../public_data/sciencespo-dataverse-ddi/
node --experimental-specifier-resolution=node -- dist/scripts/index_codebooks.js --language=fr --path=adisp-nesstar ../public_data/adisp-nesstar-ddi-repaired/
node --experimental-specifier-resolution=node -- dist/scripts/index_codebooks.js --language=fr --path=cdsp ../public_data/sciencespo-dataverse-ddi/
# Obsolete
# npx babel-node --extensions ".ts" -- src/scripts/index_codebooks.ts --language=fr --path=cdsp-obsolete ../public_data/cdsp-nesstar-ddi/
npx babel-node --extensions ".ts" -- src/scripts/index_codebooks.ts --language=fr --path=ined/gpgsurvey ../public_data/ined-gpgsurvey-nesstar-ddi/
npx babel-node --extensions ".ts" -- src/scripts/index_codebooks.ts --language=en --path=ukdataservice ../public_data/ukdataservice-nesstar-ddi/
npx babel-node --extensions ".ts" -- src/scripts/index_codebooks.ts --language=no --path=nsddata ../public_data/nsddata-nesstar-ddi/
# node --experimental-specifier-resolution=node -- dist/scripts/index_codebooks.js --language=fr --path=cdsp-obsolete ../public_data/cdsp-nesstar-ddi/
node --experimental-specifier-resolution=node -- dist/scripts/index_codebooks.js --language=fr --path=ined/gpgsurvey ../public_data/ined-gpgsurvey-nesstar-ddi/
node --experimental-specifier-resolution=node -- dist/scripts/index_codebooks.js --language=en --path=ukdataservice ../public_data/ukdataservice-nesstar-ddi/
node --experimental-specifier-resolution=node -- dist/scripts/index_codebooks.js --language=no --path=nsddata ../public_data/nsddata-nesstar-ddi/
```
### Extracting Words from CodeBooks for Autocompletion
```bash
npx babel-node --extensions ".ts" -- src/scripts/index_words.ts
node --experimental-specifier-resolution=node -- dist/scripts/index_words.js
```
## Development
......@@ -133,11 +139,11 @@ npx babel-node --extensions ".ts" -- src/scripts/index_words.ts
#### Extracting TypeScript Raw Types from Progedo DDI Files
```bash
npx babel-node --extensions ".ts" --max-old-space-size=10240 -- src/scripts/raw_types_from_ddi_files.ts ../public_data/adisp-oai-pmh-ddi-repaired/ ../public_data/sciencespo-dataverse-ddi/ ../public_data/ined-nesstar-ddi/
node --experimental-specifier-resolution=node --max-old-space-size=10240 -- dist/scripts/raw_types_from_ddi_files.js ../public_data/adisp-oai-pmh-ddi-repaired/ ../public_data/sciencespo-dataverse-ddi/ ../public_data/ined-nesstar-ddi/
npx babel-node --extensions ".ts" --max-old-space-size=10240 -- src/scripts/raw_types_from_ddi_files.ts ../public_data/adisp-oai-pmh-ddi-repaired/ ../public_data/sciencespo-dataverse-ddi/ ../public_data/ined-nesstar-ddi/ --version=1.2.2
npx babel-node --extensions ".ts" --max-old-space-size=10240 -- src/scripts/raw_types_from_ddi_files.ts ../public_data/adisp-oai-pmh-ddi-repaired/ ../public_data/sciencespo-dataverse-ddi/ ../public_data/ined-nesstar-ddi/ --version=1.3
npx babel-node --extensions ".ts" --max-old-space-size=10240 -- src/scripts/raw_types_from_ddi_files.ts ../public_data/adisp-oai-pmh-ddi-repaired/ ../public_data/sciencespo-dataverse-ddi/ ../public_data/ined-nesstar-ddi/ --version=2.5
node --experimental-specifier-resolution=node --max-old-space-size=10240 -- dist/scripts/raw_types_from_ddi_files.js ../public_data/adisp-oai-pmh-ddi-repaired/ ../public_data/sciencespo-dataverse-ddi/ ../public_data/ined-nesstar-ddi/ --version=1.2.2
node --experimental-specifier-resolution=node --max-old-space-size=10240 -- dist/scripts/raw_types_from_ddi_files.js ../public_data/adisp-oai-pmh-ddi-repaired/ ../public_data/sciencespo-dataverse-ddi/ ../public_data/ined-nesstar-ddi/ --version=1.3
node --experimental-specifier-resolution=node --max-old-space-size=10240 -- dist/scripts/raw_types_from_ddi_files.js ../public_data/adisp-oai-pmh-ddi-repaired/ ../public_data/sciencespo-dataverse-ddi/ ../public_data/ined-nesstar-ddi/ --version=2.5
# Prettify generated TypeScript file:
npm run prettier
......@@ -146,10 +152,10 @@ npm run prettier
#### Extracting TypeScript Raw Types for Other Tests
```bash
npx babel-node --extensions ".ts" --max-old-space-size=8192 -- src/scripts/raw_types_from_ddi_files.ts ../public_data/adisp-manual-ddi/ --target=src/raw_types/codebooks_adisp_manual.ts
npx babel-node --extensions ".ts" --max-old-space-size=8192 -- src/scripts/raw_types_from_ddi_files.ts ../public_data/adisp-nesstar-ddi/ --target=src/raw_types/codebooks_adisp_nesstar.ts
npx babel-node --extensions ".ts" -- src/scripts/raw_types_from_ddi_files.ts ../public_data/sciencespo-dataverse-ddi/ --target=src/raw_types/codebooks_sciencespo_dataverse.ts
npx babel-node --extensions ".ts" -- src/scripts/raw_types_from_ddi_files.ts ../public_data/ined-nesstar-ddi/ --target=src/raw_types/codebooks_ined_nesstar.ts
node --experimental-specifier-resolution=node --max-old-space-size=8192 -- dist/scripts/raw_types_from_ddi_files.js ../public_data/adisp-manual-ddi/ --target=src/raw_types/codebooks_adisp_manual.ts
node --experimental-specifier-resolution=node --max-old-space-size=8192 -- dist/scripts/raw_types_from_ddi_files.js ../public_data/adisp-nesstar-ddi/ --target=src/raw_types/codebooks_adisp_nesstar.ts
node --experimental-specifier-resolution=node -- dist/scripts/raw_types_from_ddi_files.js ../public_data/sciencespo-dataverse-ddi/ --target=src/raw_types/codebooks_sciencespo_dataverse.ts
node --experimental-specifier-resolution=node -- dist/scripts/raw_types_from_ddi_files.js ../public_data/ined-nesstar-ddi/ --target=src/raw_types/codebooks_ined_nesstar.ts
# Prettify generated TypeScript files:
npm run prettier
......
module.exports = {
presets: [
[
"@babel/preset-env",
{
bugfixes: true,
corejs: 3,
targets: "> 0.25%, not dead",
useBuiltIns: "usage",
},
],
"@babel/preset-typescript",
],
plugins: [
"@babel/plugin-proposal-class-properties",
"@babel/plugin-syntax-dynamic-import",
],
}
......@@ -10,8 +10,10 @@
"hasInstallScript": true,
"license": "AGPL-3.0-or-later",
"dependencies": {
"command-line-args": "^5.2.0",
"fast-xml-parser": "^3.17.5",
"he": "^1.2.0",
"node-stream-zip": "^1.14.0",
"slug": "^5.0.1",
"ws": "^8.1.0"
},
......@@ -26,6 +28,7 @@
"@sveltejs/kit": "next",
"@tailwindcss/forms": "^0.3.3",
"@tailwindcss/typography": "^0.4.1",
"@types/command-line-args": "^5.2.0",
"@types/fs-extra": "^9.0.12",
"@types/he": "^1.1.2",
"@types/node-fetch": "^2.5.12",
......@@ -64,7 +67,6 @@
"svelte-json-tree": "github:eraviart/svelte-json-tree",
"svelte-preprocess": "^4.7.1",
"svelte-range-slider-pips": "github:eraviart/svelte-range-slider-pips",
"svelte2tsx": "^0.4.5",
"tailwindcss": "^2.1.1",
"tippy.js": "^6.3.1",
"tslib": "^2.0.0",
......@@ -920,6 +922,12 @@
"optional": true,
"peer": true
},
"node_modules/@types/command-line-args": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.0.tgz",
"integrity": "sha512-UuKzKpJJ/Ief6ufIaIzr3A/0XnluX7RvFgwkV89Yzvm77wCh1kFaFmqN8XEnGcN62EuHdedQjEMb8mYxFLGPyA==",
"dev": true
},
"node_modules/@types/debug": {
"version": "4.1.7",
"resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.7.tgz",
......@@ -1315,6 +1323,14 @@
"sprintf-js": "~1.0.2"
}
},
"node_modules/array-back": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/array-back/-/array-back-3.1.0.tgz",
"integrity": "sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==",
"engines": {
"node": ">=6"
}
},
"node_modules/array-union": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz",
......@@ -1676,6 +1692,20 @@
"node": ">= 0.8"
}
},
"node_modules/command-line-args": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-5.2.0.tgz",
"integrity": "sha512-4zqtU1hYsSJzcJBOcNZIbW5Fbk9BkjCp1pZVhQKoRaWL5J7N4XphDLwo8aWwdQpTugxwu+jf9u2ZhkXiqp5Z6A==",
"dependencies": {
"array-back": "^3.1.0",
"find-replace": "^3.0.0",
"lodash.camelcase": "^4.3.0",
"typical": "^4.0.0"
},
"engines": {
"node": ">=4.0.0"
}
},
"node_modules/commander": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz",
......@@ -2528,6 +2558,17 @@
"node": ">=8"
}
},
"node_modules/find-replace": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/find-replace/-/find-replace-3.0.0.tgz",
"integrity": "sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==",
"dependencies": {
"array-back": "^3.0.1"
},
"engines": {
"node": ">=4.0.0"
}
},
"node_modules/find-yarn-workspace-root": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/find-yarn-workspace-root/-/find-yarn-workspace-root-2.0.0.tgz",
......@@ -3184,6 +3225,11 @@
"integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==",
"dev": true
},
"node_modules/lodash.camelcase": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
"integrity": "sha1-soqmKIorn8ZRA1x3EfZathkDMaY="
},
"node_modules/lodash.castarray": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/lodash.castarray/-/lodash.castarray-4.4.0.tgz",
......@@ -3242,15 +3288,6 @@
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/lower-case": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/lower-case/-/lower-case-2.0.2.tgz",
"integrity": "sha512-7fm3l3NAF9WfN6W3JOmf5drwpVqX78JtoGJ3A6W0a6ZnldM41w2fV5D490psKFTpMds8TJse/eHLFFsNHHjHgg==",
"dev": true,
"dependencies": {
"tslib": "^2.0.3"
}
},
"node_modules/lru-cache": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
......@@ -3929,16 +3966,6 @@
"integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==",
"dev": true
},
"node_modules/no-case": {
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/no-case/-/no-case-3.0.4.tgz",
"integrity": "sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg==",
"dev": true,
"dependencies": {
"lower-case": "^2.0.2",
"tslib": "^2.0.3"
}
},
"node_modules/node-emoji": {
"version": "1.11.0",
"resolved": "https://registry.npmjs.org/node-emoji/-/node-emoji-1.11.0.tgz",
......@@ -3963,6 +3990,18 @@
"integrity": "sha512-Qe5OUajvqrqDSy6wrWFmMwfJ0jVgwiw4T3KqmbTcZ62qW0gQkheXYhcFM1+lOVcGUoRxcEcfyvFMAnDgaF1VWw==",
"dev": true
},
"node_modules/node-stream-zip": {
"version": "1.14.0",
"resolved": "https://registry.npmjs.org/node-stream-zip/-/node-stream-zip-1.14.0.tgz",
"integrity": "sha512-SKXyiBy9DBemsPHf/piHT00Y+iPK+zwru1G6+8UdOBzITnmmPMHYBMV6M1znyzyhDhUFQW0HEmbGiPqtp51M6Q==",
"engines": {
"node": ">=0.10.0"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/antelle"
}
},
"node_modules/normalize-path": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
......@@ -4119,16 +4158,6 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/pascal-case": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/pascal-case/-/pascal-case-3.1.2.tgz",
"integrity": "sha512-uWlGT3YSnK9x3BQJaOdcZwrnV6hPpd8jFH1/ucpiLRPh/2zCVJKS19E4GvYHvaCcACn3foXZ0cLB9Wrx1KGe5g==",
"dev": true,
"dependencies": {
"no-case": "^3.0.4",
"tslib": "^2.0.3"
}
},
"node_modules/passport": {
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/passport/-/passport-0.4.1.tgz",
......@@ -5797,20 +5826,6 @@
"dev": true,
"license": "MPL-2.0"
},
"node_modules/svelte2tsx": {
"version": "0.4.5",
"resolved": "https://registry.npmjs.org/svelte2tsx/-/svelte2tsx-0.4.5.tgz",
"integrity": "sha512-5SbnH3rGA5eymeSidlPZ3qxL0P6SfJv0pWaoTVuJDxpHbI00dUZpuDTNn/WZ8yMXfU7vjY/QxKC/iYiDTbJEjg==",
"dev": true,
"dependencies": {
"dedent-js": "^1.0.1",
"pascal-case": "^3.1.1"
},
"peerDependencies": {
"svelte": "^3.24",
"typescript": "^4.1.2"
}
},
"node_modules/svgo": {
"version": "2.4.0",
"resolved": "https://registry.npmjs.org/svgo/-/svgo-2.4.0.tgz",
......@@ -6151,6 +6166,14 @@
"node": ">=4.2.0"
}
},
"node_modules/typical": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/typical/-/typical-4.0.0.tgz",
"integrity": "sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw==",
"engines": {
"node": ">=8"
}
},
"node_modules/uniqs": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/uniqs/-/uniqs-2.0.0.tgz",
......@@ -7035,6 +7058,12 @@
"optional": true,
"peer": true
},
"@types/command-line-args": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.0.tgz",
"integrity": "sha512-UuKzKpJJ/Ief6ufIaIzr3A/0XnluX7RvFgwkV89Yzvm77wCh1kFaFmqN8XEnGcN62EuHdedQjEMb8mYxFLGPyA==",
"dev": true
},
"@types/debug": {
"version": "4.1.7",
"resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.7.tgz",
......@@ -7326,6 +7355,11 @@
"sprintf-js": "~1.0.2"
}
},
"array-back": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/array-back/-/array-back-3.1.0.tgz",
"integrity": "sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q=="
},
"array-union": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz",
......@@ -7600,6 +7634,17 @@
"delayed-stream": "~1.0.0"
}
},
"command-line-args": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-5.2.0.tgz",
"integrity": "sha512-4zqtU1hYsSJzcJBOcNZIbW5Fbk9BkjCp1pZVhQKoRaWL5J7N4XphDLwo8aWwdQpTugxwu+jf9u2ZhkXiqp5Z6A==",
"requires": {
"array-back": "^3.1.0",
"find-replace": "^3.0.0",
"lodash.camelcase": "^4.3.0",
"typical": "^4.0.0"
}
},
"commander": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz",
......@@ -8243,6 +8288,14 @@
"to-regex-range": "^5.0.1"
}
},
"find-replace": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/find-replace/-/find-replace-3.0.0.tgz",
"integrity": "sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==",
"requires": {
"array-back": "^3.0.1"
}
},
"find-yarn-workspace-root": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/find-yarn-workspace-root/-/find-yarn-workspace-root-2.0.0.tgz",
......@@ -8741,6 +8794,11 @@
"integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==",
"dev": true
},
"lodash.camelcase": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
"integrity": "sha1-soqmKIorn8ZRA1x3EfZathkDMaY="
},
"lodash.castarray": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/lodash.castarray/-/lodash.castarray-4.4.0.tgz",
......@@ -8795,15 +8853,6 @@
"integrity": "sha512-XhUjWR5CFaQ03JOP+iSDS9koy8T5jfoImCZ4XprElw3BXsSk4MpVYOLw/6LTDKZhO13PlAXnB5gS4MHQTpkSOw==",
"dev": true
},
"lower-case": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/lower-case/-/lower-case-2.0.2.tgz",
"integrity": "sha512-7fm3l3NAF9WfN6W3JOmf5drwpVqX78JtoGJ3A6W0a6ZnldM41w2fV5D490psKFTpMds8TJse/eHLFFsNHHjHgg==",
"dev": true,
"requires": {
"tslib": "^2.0.3"
}
},
"lru-cache": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
......@@ -9219,16 +9268,6 @@
"integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==",
"dev": true
},
"no-case": {
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/no-case/-/no-case-3.0.4.tgz",
"integrity": "sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg==",
"dev": true,
"requires": {
"lower-case": "^2.0.2",
"tslib": "^2.0.3"
}
},
"node-emoji": {
"version": "1.11.0",
"resolved": "https://registry.npmjs.org/node-emoji/-/node-emoji-1.11.0.tgz",
......@@ -9250,6 +9289,11 @@
"integrity": "sha512-Qe5OUajvqrqDSy6wrWFmMwfJ0jVgwiw4T3KqmbTcZ62qW0gQkheXYhcFM1+lOVcGUoRxcEcfyvFMAnDgaF1VWw==",
"dev": true
},
"node-stream-zip": {
"version": "1.14.0",
"resolved": "https://registry.npmjs.org/node-stream-zip/-/node-stream-zip-1.14.0.tgz",
"integrity": "sha512-SKXyiBy9DBemsPHf/piHT00Y+iPK+zwru1G6+8UdOBzITnmmPMHYBMV6M1znyzyhDhUFQW0HEmbGiPqtp51M6Q=="
},
"normalize-path": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
......@@ -9363,16 +9407,6 @@
"lines-and-columns": "^1.1.6"
}
},
"pascal-case": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/pascal-case/-/pascal-case-3.1.2.tgz",
"integrity": "sha512-uWlGT3YSnK9x3BQJaOdcZwrnV6hPpd8jFH1/ucpiLRPh/2zCVJKS19E4GvYHvaCcACn3foXZ0cLB9Wrx1KGe5g==",
"dev": true,
"requires": {
"no-case": "^3.0.4",
"tslib": "^2.0.3"
}
},
"passport": {
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/passport/-/passport-0.4.1.tgz",
......@@ -10508,16 +10542,6 @@
"dev": true,
"from": "svelte-range-slider-pips@github:eraviart/svelte-range-slider-pips"
},
"svelte2tsx": {
"version": "0.4.5",
"resolved": "https://registry.npmjs.org/svelte2tsx/-/svelte2tsx-0.4.5.tgz",
"integrity": "sha512-5SbnH3rGA5eymeSidlPZ3qxL0P6SfJv0pWaoTVuJDxpHbI00dUZpuDTNn/WZ8yMXfU7vjY/QxKC/iYiDTbJEjg==",
"dev": true,
"requires": {
"dedent-js": "^1.0.1",
"pascal-case": "^3.1.1"
}
},
"svgo": {
"version": "2.4.0",
"resolved": "https://registry.npmjs.org/svgo/-/svgo-2.4.0.tgz",
......@@ -10775,6 +10799,11 @@
"integrity": "sha512-DqQgihaQ9cUrskJo9kIyW/+g0Vxsk8cDtZ52a3NGh0YNTfpUSArXSohyUGnvbPazEPLu398C0UxmKSOrPumUzA==",
"dev": true
},
"typical": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/typical/-/typical-4.0.0.tgz",
"integrity": "sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw=="
},
"uniqs": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/uniqs/-/uniqs-2.0.0.tgz",
......
......@@ -10,19 +10,23 @@
"author": "DBnomics Team",
"scripts": {
"build": "svelte-kit build",
"build-scripts": "npm run build-scripts:lib && npm run build-scripts:tsc",
"build-scripts:lib": "mkdir -p dist/node_modules && cd dist/node_modules/ && rm -f '$lib' && ln -s ../lib/ '$lib'",
"build-scripts:tsc": "tsc --declaration --outDir dist --rootDir src",
"check": "svelte-check --tsconfig ./tsconfig.json",
"check:watch": "svelte-check --tsconfig ./tsconfig.json --watch",
"configure": "babel-node --extensions \".ts\" src/configure.ts",
"configure": "node --experimental-specifier-resolution=node dist/scripts/configure.js",
"dev": "svelte-kit dev",
"format": "prettier --write --plugin-search-dir=. .",
"lint": "prettier --check --plugin-search-dir=. . && eslint --ignore-path .gitignore .",
"package": "svelte-kit package",
"postinstall": "patch-package",
"preview": "svelte-kit preview"
},
"dependencies": {
"command-line-args": "^5.2.0",
"fast-xml-parser": "^3.17.5",
"he": "^1.2.0",
"node-stream-zip": "^1.14.0",
"slug": "^5.0.1",
"ws": "^8.1.0"
},
......@@ -37,6 +41,7 @@
"@sveltejs/kit": "next",
"@tailwindcss/forms": "^0.3.3",
"@tailwindcss/typography": "^0.4.1",
"@types/command-line-args": "^5.2.0",
"@types/fs-extra": "^9.0.12",
"@types/he": "^1.1.2",
"@types/node-fetch": "^2.5.12",
......@@ -75,7 +80,6 @@
"svelte-json-tree": "github:eraviart/svelte-json-tree",
"svelte-preprocess": "^4.7.1",
"svelte-range-slider-pips": "github:eraviart/svelte-range-slider-pips",
"svelte2tsx": "^0.4.5",
"tailwindcss": "^2.1.1",
"tippy.js": "^6.3.1",
"tslib": "^2.0.0",
......
import {
Audit,
auditBoolean,
auditOptions,
auditRequire,
auditSetNullish,
auditTrimString,
cleanAudit,
} from "@auditors/core"
import commandLineArgs from "command-line-args"
import { Language } from "$lib/data"
import { connectDb } from "$lib/server/database.js"
import { indexCodeBooks } from "$lib/server/indexers.js"
const optionsDefinitions = [
{
alias: "l",
help: "default language for CodeBooks",
name: "language",
type: String,
},
{
alias: "p",
help: "path of group containing CodeBooks present in directory",
name: "path",
type: String,
},
{
alias: "s",
help: "don't log anything",
name: "silent",
type: Boolean,
},
{
alias: "v",
help: "verbose logs",
name: "verbose",
type: Boolean,
},
{
defaultOption: true,
help: "directory containing XML DDI CodeBook files",
name: "xmlDir",
type: String,