Commit 6068fe87 authored by Emmanuel Raviart's avatar Emmanuel Raviart
Browse files

Add first (very minimal) UI.

parent 6955ed1c
Pipeline #211014 failed with stage
in 2 minutes and 29 seconds
......@@ -10,6 +10,42 @@ cd ez-ddi/
npm install
```
### Database Creation
#### Using _Debian GNU/Linux_
As `root` user:
```bash
apt install postgresql
su - postgres
psql
```
#### Using _MacOS_
```bash
brew install postgresql
psql postgres
```
#### For everybody
```sql
CREATE USER ezddi WITH PASSWORD 'ezddi';
CREATE DATABASE ezddi WITH OWNER ezddi;
\connect ezddi
CREATE EXTENSION IF NOT EXISTS pg_trgm;
\q
logout # For Debian only
```
As normal user, create database tables:
```bash
npm run configure
```
## Usage
### Fetching Nesstar Servers
......@@ -33,7 +69,7 @@ npx babel-node --extensions ".ts" src/scripts/retrieve_nesstar_ddis.js --url htt
### Extracting TypeScript Raw Types from DDI files
```bash
npx babel-node --extensions ".ts" --max-old-space-size=10240 src/scripts/raw_types_from_ddi_files.ts ../public_data/adisp-ddi/ ../public_data/adisp-ddi/ ../public_data/ined-ddi/ --target=src/raw_types/enquetes.ts
npx babel-node --extensions ".ts" --max-old-space-size=10240 src/scripts/raw_types_from_ddi_files.ts ../public_data/adisp-ddi/ ../public_data/adisp-ddi/ ../public_data/ined-ddi/ --target=src/raw_types/code_books.ts
npx babel-node --extensions ".ts" --max-old-space-size=8192 src/scripts/raw_types_from_ddi_files.ts ../public_data/xml-ddi-adisp/ --target=src/raw_types/enquetes_adisp_fournies.ts
npx babel-node --extensions ".ts" --max-old-space-size=8192 src/scripts/raw_types_from_ddi_files.ts ../public_data/adisp-ddi/ --target=src/raw_types/enquetes_adisp.ts
npx babel-node --extensions ".ts" src/scripts/raw_types_from_ddi_files.ts ../public_data/cdsp-ddi/ --target=src/raw_types/enquetes_cdsp.ts
......@@ -42,3 +78,11 @@ npx babel-node --extensions ".ts" src/scripts/raw_types_from_ddi_files.ts ../pub
# Prettify generated TypeScript files:
npm run prettier
```
### Indexing DDI files
```bash
npx babel-node --extensions ".ts" -- src/scripts/index_code_books.ts --path=adisp-ddi --title=\"Archives de données issues de la statistique publique \(ADISP\)\" ../public_data/
npx babel-node --extensions ".ts" -- src/scripts/index_code_books.ts --path=cdsp-ddi --title=\"SciencesPo Centre de données socio-politiques \(CDSP\)\" ../public_data/
npx babel-node --extensions ".ts" -- src/scripts/index_code_books.ts --path=ined-ddi --title=\"Institut national d\'études démographiques \(INED\)\" ../public_data/
```
......@@ -11,5 +11,8 @@ module.exports = {
],
"@babel/preset-typescript",
],
plugins: ["@babel/plugin-syntax-dynamic-import"],
plugins: [
"@babel/plugin-proposal-class-properties",
"@babel/plugin-syntax-dynamic-import",
],
}
......@@ -23,6 +23,7 @@
"devDependencies": {
"@babel/core": "^7.0.0",
"@babel/node": "^7.12.10",
"@babel/plugin-proposal-class-properties": "^7.12.1",
"@babel/plugin-syntax-dynamic-import": "^7.0.0",
"@babel/plugin-transform-runtime": "^7.0.0",
"@babel/preset-env": "^7.0.0",
......
<script lang="ts">
import type { Node } from "../data"
import { localize } from "../stores"
export let node: Node
$: _ = $localize
$: codeBook = node.codeBook
$: keywords = codeBook?.stdyDscr?.stdyInfo.subject?.keyword
</script>
{#if codeBook != null}
<div class="mx-auto">
<!-- <h2 class="font-bold text-2xl">
{codeBook.stdyDscr.citation.titlStmt.titl}
</h2> -->
<p>{codeBook.stdyDscr.stdyInfo.abstract}</p>
{#if keywords}
{_('Keywords')}{_('colon')}
:
{#each keywords as keyword, index}
{#if index > 0}, {/if}{keyword}
{/each}
{/if}
<pre>{JSON.stringify(codeBook, null, 2)}</pre>
</div>
{/if}
<script lang="ts">
import { stores } from "@sapper/app"
import Pagination from "./Pagination.svelte"
import SearchForm from "./SearchForm.svelte"
import type { Node } from "../data"
import { localize } from "../stores"
export let node: Node
const { page } = stores()
$: _ = $localize
$: children = node.children!
$: childrenCount = node.childrenCount!
$: query = $page.query
$: term = query.q
</script>
<div class="mx-auto">
<SearchForm placeholder="niveau de vie…" searchPath={$page.path} {term} />
<ul>
{#each children as child}
<li>
<a class="button ~info !low" href={child.path}>{child.title}
({_(child.type)})</a>
</li>
{/each}
</ul>
<Pagination
count={childrenCount}
currentPageCount={children.length}
queryParams={query}
url={$page.path} />
</div>
<script lang="ts">
import { stores } from "@sapper/app"
import { localize } from "../stores"
// import { stores } from "@sapper/app"
// import { localize } from "../stores"
export let segment: string | undefined
const { session } = stores()
// const { session } = stores()
$: _ = $localize
// $: _ = $localize
$: roles = $session.roles
// $: roles = $session.roles
</script>
<nav
<!-- <nav
class="flext flex-wrap flex-grow w-full max-w-full px-2 py-4 rounded-lg shadow-lg md:w-auto md:items-center md:justify-between bg-neutral-000 md:bg-transparent md:shadow-none md:p-0">
<ul class="flex flex-wrap items-center">
{#if roles.has('ezddi_diffuseur') || roles.has('ezddi_support')}
......@@ -44,4 +44,4 @@
</li>
{/if}
</ul>
</nav>
</nav> -->
<script lang="ts">
import CodeBookView from "./CodeBookView.svelte"
import GroupView from "./GroupView.svelte"
import type { Node } from "../data"
import { assertNeverNodeType, NodeType } from "../data"
export let mode = "full" // "embed", "full", "preview"
export let node: Node
$: component = componentFromNodeType(node.type)
function componentFromNodeType(type: NodeType) {
switch (type) {
case NodeType.CodeBook:
return CodeBookView
case NodeType.Group:
return GroupView
default:
assertNeverNodeType(type)
}
}
</script>
<svelte:component this={component} {mode} {node} />
<script>
<script lang="ts">
import { goto } from "@sapper/app"
import { buildSearchUrl } from "../urls"
import { localize } from "../stores"
import { newNodeLocalUrl } from "../urls"
export let limit = 10
export let offset = 0
export let placeholder = null
export let placeholder: string | undefined
export let searchPath = ""
export let term = ""
$: _ = $localize
function search() {
goto(buildSearchUrl(searchPath, { limit, offset, term }))
goto(
newNodeLocalUrl(searchPath, {
offset,
term,
}),
)
}
</script>
......@@ -21,7 +28,7 @@
{placeholder}
type="text"
bind:value={term} />
<button class="btn-blue ml-2" type="submit">Search</button>
<button class="btn-blue ml-2" type="submit">{_('Search')}</button>
</div>
</form>
</section>
import assert from "assert"
import dedent from "dedent-js"
import { Node, NodeType } from "./data"
import { db, versionNumber } from "./database"
import { indexNode } from "./indexers"
async function configureDatabase() {
export async function configure(): Promise<void> {
await configureDatabase()
const rootNode: Node = {
title: "Root",
path: "",
type: NodeType.Group,
}
await indexNode(rootNode)
}
async function configureDatabase(): Promise<void> {
// Check that database exists.
await db.connect()
......@@ -36,6 +49,23 @@ async function configureDatabase() {
// Types
// Enum: node_type
try {
await db.none(
dedent`
CREATE TYPE node_type AS ENUM (
'CodeBook',
'Group'
)
`,
)
} catch (e) {
// 42710: type "chart_node_type" already exists
if (e.code !== "42710") {
throw e
}
}
// Table: users
await db.none(
dedent`
......@@ -48,6 +78,29 @@ async function configureDatabase() {
// Tables
// Table: nodes
await db.none(
dedent`
CREATE TABLE IF NOT EXISTS nodes (
file_path text,
path text NOT NULL PRIMARY KEY,
title text NOT NULL,
type node_type NOT NULL
)
`,
)
// Table: nodes_autocomplete
await db.none(
dedent`
CREATE TABLE IF NOT EXISTS nodes_autocomplete (
autocomplete text NOT NULL,
path text NOT NULL REFERENCES nodes(path) ON DELETE CASCADE,
PRIMARY KEY (path, autocomplete)
)
`,
)
// Table: sessions
// Cf node_modules/connect-pg-simple/table.sql
await db.none(
......@@ -62,7 +115,35 @@ async function configureDatabase() {
// Apply patches that must be executed after every table is created.
// Add indexes once every table and column exists.
await db.none(
dedent`
CREATE INDEX IF NOT EXISTS nodes_autocomplete_trigrams_idx
ON nodes_autocomplete
USING GIST (autocomplete gist_trgm_ops)
`,
)
// Add comments once every table and column exists.
for (const command of [
"COMMENT ON TABLE nodes IS 'nodes (of the tree of nodes'",
"COMMENT ON COLUMN nodes.file_path IS 'relative path of the file containing the" +
" data of the node'",
'COMMENT ON COLUMN nodes.path IS \'"/"-separated concatenation of' +
" the segments of the node and its ancestors'",
"COMMENT ON COLUMN nodes.title IS 'name of node'",
"COMMENT ON COLUMN nodes.type IS 'type of node'",
"COMMENT ON TABLE nodes_autocomplete IS 'autocompletions texts for nodes'",
"COMMENT ON COLUMN nodes_autocomplete.autocomplete IS 'autocompletion text of node'",
"COMMENT ON COLUMN nodes_autocomplete.path IS 'path of node'",
"COMMENT ON TABLE version IS 'version of database'",
"COMMENT ON COLUMN version.number IS 'version number of database schema'",
]) {
await db.none(command)
}
// Upgrade version number if needed.
......@@ -81,7 +162,7 @@ async function configureDatabase() {
}
}
configureDatabase()
configure()
.then(() => process.exit(0))
.catch((error) => {
console.log(error.stack || error)
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
......@@ -95,53 +95,21 @@ export function quietRetrievalError<T extends Errorwise>(
}
}
export async function retrieveDataItem(
export async function retrieveNode(
fetch: (input: RequestInfo, init?: RequestInit) => Promise<Response>,
url: string,
{
deep = false,
errorHandler = logRetrievalError,
follow = new Set(),
location = "retrieveDataItem",
}: {
errorHandler?: ErrorHandler<JsonData>
follow?: Set<Follow>
location?: string
} = {},
): Promise<any> {
const queryParams: {
follow: string[]
} = {
follow: [...follow].sort(),
}
const query = stringifyQuery(queryParams)
url = `${url}${query ? "?" + query : ""}`
const headers: { [key: string]: string } = {}
const response = await fetch(url, {
credentials: "include",
headers,
})
const [json, error] = await validateJsonResponse(null)(response)
// json.url = url
// json.follow = [...follow].sort()
if (error !== null) {
return errorHandler(location, url, response, json, error)
}
return fromJson(json, follow)
}
export async function retrieveDataList(
fetch: (input: RequestInfo, init?: RequestInit) => Promise<Response>,
url: string,
{
errorHandler = logRetrievalError,
follow = new Set(),
follow = [],
limit,
location = "retrieveDataList",
offset,
term,
}: {
deep?: boolean
errorHandler?: ErrorHandler<JsonData>
follow?: Set<Follow>
follow?: Iterable<Follow>
limit?: number | string
location?: string
offset?: number | string
......@@ -149,6 +117,7 @@ export async function retrieveDataList(
} = {},
): Promise<any> {
const queryParams: {
deep?: boolean
follow: string[]
limit?: number
offset?: number
......@@ -156,6 +125,10 @@ export async function retrieveDataList(
} = {
follow: [...follow].sort(),
}
console.log("deep", deep)
if (deep) {
queryParams.deep = true
}
if (typeof limit === "string") {
limit = parseInt(limit)
if (isNaN(limit)) {
......@@ -186,6 +159,7 @@ export async function retrieveDataList(
})
const [json, error] = await validateJsonResponse(null)(response)
// json.url = url
// json.deep = deep
// json.follow = [...follow].sort()
// json.limit = limit
// json.location = location
......@@ -193,5 +167,5 @@ export async function retrieveDataList(
if (error !== null) {
return errorHandler(location, url, response, json, error)
}
return fromJson(json, follow)
return fromJson(json, new Set(follow))
}
import fs from "fs-extra"
import path from "path"
export function* walkDir(
rootDir: string,
relativeSplitDir: string[] = [],
): Generator<string[]> {
const dir = path.join(rootDir, ...relativeSplitDir)
for (const filename of fs.readdirSync(dir)) {
if (filename[0] === ".") {
continue
}
const filePath = path.join(dir, filename)
const relativeSplitPath = [...relativeSplitDir, filename]
if (fs.statSync(filePath).isDirectory()) {
yield* walkDir(rootDir, relativeSplitPath)
} else {
yield relativeSplitPath
}
}
}
import dedent from "dedent-js"
import xmlParser from "fast-xml-parser"
import fs from "fs-extra"
import path from "path"
import { db } from "./database"
import { Node, NodeType } from "./data"
import { walkDir } from "./file_systems"
import type { CodeBook } from "./raw_types/code_books"
class CodeBooksIndexer {
existingAutocompletes: Set<string> = new Set()
existingPaths: Set<string> = new Set()
readonly path: string
// Doesn't work, because of "path" module:
// constructor(public readonly path: string, readonly types?: Set<NodeType>) {}
constructor(path: string, readonly types?: Set<NodeType>) {
this.path = path
}
/// Retrieve existing codeBooks in database.
async start(): Promise<void> {
const whereClauses = [`path ~ '^$<path:value>($|/)'`]
if (this.types !== undefined && this.types.size > 0) {
whereClauses.push("type IN ($<types:list>)")
}
const whereClause =
whereClauses.length > 0 ? "WHERE " + whereClauses.join(" AND ") : ""
const paths = await db.map(
dedent`
SELECT path
FROM nodes
${whereClause}
`,
{
path: this.path,
types: [...(this.types ?? [])],
},
({ path }: { path: string }) => path,
)
for (const path of paths) {
this.existingPaths.add(path)
}
const autocompletes = await db.map(
dedent`
SELECT autocomplete, path
FROM nodes_autocomplete
WHERE path IN (
SELECT path
FROM nodes
${whereClause}
)
`,
{
path: this.path,
types: [...(this.types ?? [])],
},
({ autocomplete, path }: { autocomplete: string; path: string }) =>
`${path}|${autocomplete}`,
)
for (const autocomplete of autocompletes) {
this.existingAutocompletes.add(autocomplete)
}
}
/// Delete obsolete nodes from database.
async stop(): Promise<void> {
for (const key of this.existingAutocompletes) {
const [path, autocomplete] = key.split("|")
await db.none(
dedent`
DELETE FROM nodes_autocomplete
WHERE
autocomplete = $<autocomplete>
AND path = $<path>
`,
{
autocomplete,
path,
},
)
}
for (const path of this.existingPaths) {
await db.none(
dedent`
DELETE FROM nodes
WHERE
path = $<path>
`,
{
path,
},
)
}
}
async upsertCodeBook(codeBook: CodeBook, filePath: string): Promise<void> {
const path = [this.path, codeBook["@ID"]].join("/")
const title = codeBook.stdyDscr.citation.titlStmt.titl
await db.none(
dedent`
INSERT INTO nodes (
file_path,
path,
title,
type
)
VALUES (
$<filePath>,
$<path>,
$<title>,
$<type>
)
ON CONFLICT (path)
DO UPDATE SET
file_path = $<filePath>,
title = $<title>,
type = $<type>
`,
{
filePath,
path,
title,
type: NodeType.CodeBook,
},
)
this.existingPaths.delete(path)
const autocomplete = title
await db.none(
dedent`