Commit 03db14d1 authored by Emmanuel Raviart's avatar Emmanuel Raviart
Browse files

0.4

parent f0f99284
Pipeline #226969 failed with stage
in 11 minutes
......@@ -149,3 +149,25 @@ npx babel-node --extensions ".ts" -- src/scripts/raw_types_from_ddi_files.ts ../
# Prettify generated TypeScript files:
npm run prettier
```
### Generating a CSV file of all organizations
```sql
COPY (
SELECT name, acronym, relation
FROM organizations
INNER JOIN study_organization_associations
ON organizations.name = study_organization_associations.organization_name
GROUP BY name, acronym, relation
ORDER BY name, relation
) TO '/tmp/organizations.csv' DELIMITER ',' CSV HEADER;
COPY (
SELECT name, acronym, relation, study_path
FROM organizations
INNER JOIN study_organization_associations
ON organizations.name = study_organization_associations.organization_name
GROUP BY name, acronym, relation, study_path
ORDER BY name, relation
) TO '/tmp/organizations_with_studies.csv' DELIMITER ',' CSV HEADER;
```
......@@ -32,4 +32,9 @@ PUBLIC_DATA_DIR="../public_data"
# Change value OF SESSION_SECRET!
SESSION_SECRET="SESSION_SECRET"
# Directory containing Markdown files used by application (default).
TEXTS_DIR="../data-catalogue-textes"
# or (default) use an URL to retrieve Markdown files from a remote website:
# TEXTS_DIR=https://git.nomics.world/progedo/data-catalogue-textes/-/raw/master/
TITLE="Réseau Quételet (dev)"
This diff is collapsed.
......@@ -6,7 +6,7 @@
"type": "git",
"url": "https://git.nomics.world/progedo/data-catalogue.git"
},
"version": "0.3.2",
"version": "0.4",
"author": "DBnomics Team",
"scripts": {
"build": "sapper build --legacy",
......@@ -59,6 +59,7 @@
"@rollup/plugin-replace": "^2.2.0",
"@rollup/plugin-typescript": "^8.1.0",
"@rollup/plugin-url": "^6.0.0",
"@tailwindcss/typography": "^0.4.0",
"@tsconfig/svelte": "^1.0.10",
"@types/command-line-args": "^5.0.0",
"@types/compression": "^1.7.0",
......@@ -84,8 +85,10 @@
"dotenv": "^8.2.0",
"fa-svelte": "^3.1.0",
"fast-xml-parser": "^3.17.4",
"front-matter": "^4.0.2",
"fs-extra": "^9.0.1",
"kleur": "^4.1.3",
"mdast-util-from-markdown": "^0.8.5",
"morgan": "^1.10.0",
"openid-client": "^4.2.2",
"passport": "^0.4.1",
......@@ -103,6 +106,7 @@
"rollup": "^2.3.4",
"rollup-plugin-svelte": "^7.0.0",
"rollup-plugin-terser": "^7.0.0",
"sanitize-filename": "^1.6.3",
"sapper": "^0.29.0",
"slug": "^4.0.2",
"sockette": "^2.0.6",
......@@ -112,6 +116,7 @@
"svelte-preprocess": "^4.6.1",
"svelte-range-slider-pips": "github:eraviart/svelte-range-slider-pips",
"tailwindcss": "^2.0.1",
"tippy.js": "^6.3.1",
"tslib": "^2.0.1",
"typescript": "^4.1.2",
"xmldom": "^0.4.0"
......
......@@ -35,7 +35,7 @@ const onwarn = (warning, onwarn) => {
switch (warning.code) {
case "CIRCULAR_DEPENDENCY":
if (
/[/\\](@fluent|@sapper|passport|svelte-json-tree)[/\\]/.test(
/[/\\](@fluent|@sapper|mdast-util-to-hast|passport|svelte-json-tree)[/\\]/.test(
warning.message,
)
) {
......@@ -92,6 +92,7 @@ export default {
}),
commonjs({ sourceMap: !!sourcemap }),
typescript({ sourceMap: !!sourcemap }),
json(),
legacy &&
babel({
......
......@@ -5,9 +5,10 @@ import {
StudyOrganizationRelation,
StudyTermRelation,
} from "./data"
import type {
import {
CodeBook,
DataKindEnum,
Event,
ResInstru,
SerStmt,
Var,
......@@ -62,6 +63,17 @@ export function getCodeBookStudyDescription(
return undefined
}
export function getCodeBookTag(codeBook: CodeBook, address: string) {
let node = codeBook
if (!address.startsWith("codeBook")) return address
const addresses = address.split("/").slice(1) // Skip "codeBook"
for (const element of addresses) {
node = (node as any)[element]
if (node == undefined) return undefined
}
return nodeToString(node)
}
export function getCodeBookVariable(
codeBook: CodeBook,
id: string,
......@@ -195,14 +207,37 @@ function* iterCodeBookOrganizations(
export function* iterCodeBookProducers(
codeBook: CodeBook,
): Generator<Organization, void, void> {
yield* iterCodeBookOrganizations(codeBook.stdyDscr.citation.prodStmt.producer)
yield* iterCodeBookOrganizations(
codeBook.stdyDscr.citation.prodStmt?.producer,
)
}
export function* iterCodeBookTimePrdDates(
codeBook: CodeBook,
): Generator<string, void, void> {
for (const instant of codeBook.stdyDscr.stdyInfo.sumDscr?.timePrd ?? []) {
yield instant["@date"]
let timePeriod = codeBook.stdyDscr.stdyInfo.sumDscr?.timePrd
if (timePeriod === undefined) {
timePeriod = []
} else if (!Array.isArray(timePeriod)) {
timePeriod =
timePeriod instanceof Date
? [
{
"#text": timePeriod,
"@date": timePeriod.toISOString(),
"@event": Event.Single,
},
]
: [timePeriod]
}
for (const instant of timePeriod) {
let date = instant["@date"] ?? instant["#text"]
if (date instanceof Date) {
date = date.toISOString()
}
if (date !== undefined) {
yield date
}
}
}
......@@ -226,7 +261,7 @@ export function* iterCodeBookVariables(
): Generator<Var, void, void> {
const { dataDscr } = codeBook
if (typeof dataDscr !== "string") {
for (const variable of dataDscr.var ?? []) {
for (const variable of dataDscr?.var ?? []) {
yield variable
}
}
......@@ -364,3 +399,13 @@ export function* iterStudyYears(
yield* years
}
}
function nodeToString(node: any): string {
return typeof node === "number"
? node.toString()
: typeof node === "string"
? node
: Array.isArray(node)
? node.map((item) => nodeToString(item)).join(", ")
: JSON.stringify(node)
}
......@@ -70,7 +70,7 @@ export function auditConfig(audit: Audit, data: any): [any, any] {
auditSwitch([auditTrimString, auditStringToBoolean], auditBoolean),
auditSetNullish(false),
)
for (const key of ["publicDataDir", "sessionSecret", "title"]) {
for (const key of ["publicDataDir", "sessionSecret", "textsDir", "title"]) {
audit.attribute(
data,
key,
......
import {
Audit,
auditArray,
auditFunction,
auditRequire,
auditTrimString,
strictAudit,
} from "@auditors/core"
import sanitizeFilename from "sanitize-filename"
export function auditMarkdownParameters(audit: Audit, data: any): [any, any] {
if (data == null) {
return [data, null]
}
if (typeof data !== "object") {
return audit.unexpectedType(data, "object")
}
data = {
...data,
}
const errors: { [key: string]: any } = {}
const remainingKeys = new Set(Object.keys(data))
audit.attribute(
data,
"name",
true,
errors,
remainingKeys,
auditFunction(sanitizeFilename),
auditTrimString,
auditRequire,
)
return audit.reduceRemaining(data, errors, remainingKeys)
}
export function auditSeriesParameters(audit: Audit, data: any): [any, any] {
if (data == null) {
......@@ -87,6 +117,10 @@ export function auditVariableParameters(audit: Audit, data: any): [any, any] {
return audit.reduceRemaining(data, errors, remainingKeys)
}
export function validateMarkdownParameters(data: any): [any, any] {
return auditMarkdownParameters(strictAudit, data)
}
export function validateSeriesParameters(data: any): [any, any] {
return auditSeriesParameters(strictAudit, data)
}
......
......@@ -44,6 +44,20 @@ export function auditApiSearchQuery(audit: Audit, data: any): [any, any] {
remainingKeys,
auditQuerySet,
)
audit.attribute(
data,
"first_year",
true,
errors,
remainingKeys,
auditTrimString,
auditStringToNumber,
auditInteger,
auditTest(
(value: number) => value >= 1700,
"Value must be greater than or equal to 1700",
),
)
audit.attribute(
data,
"follow",
......@@ -63,6 +77,32 @@ export function auditApiSearchQuery(audit: Audit, data: any): [any, any] {
auditOptions(Object.values(Language)),
auditSetNullish(Language.En),
)
audit.attribute(
data,
"last_year",
true,
errors,
remainingKeys,
auditTrimString,
auditStringToNumber,
auditInteger,
auditTest(
(value: number) => value >= 1700,
"Value must be greater than or equal to 1700",
),
)
if (errors.first_year === undefined && errors.last_year === undefined) {
if (data.first_year !== undefined && data.last_year !== undefined) {
if (data.first_year > data.last_year) {
errors.last_year =
"Last year must be greater than or equal to first year"
}
} else if (data.first_year !== undefined) {
errors.last_year = "Missing values"
} else if (data.last_year !== undefined) {
errors.first_year = "Missing values"
}
}
audit.attribute(
data,
"limit",
......@@ -319,7 +359,47 @@ export function auditSearchQuery(audit: Audit, data: any): [any, any] {
remainingKeys,
auditQuerySet,
)
audit.attribute(
data,
"first_year",
true,
errors,
remainingKeys,
auditTrimString,
auditStringToNumber,
auditInteger,
auditTest(
(value: number) => value >= 1700,
"Value must be greater than or equal to 1700",
),
)
audit.attribute(data, "keyword", true, errors, remainingKeys, auditQuerySet)
audit.attribute(
data,
"last_year",
true,
errors,
remainingKeys,
auditTrimString,
auditStringToNumber,
auditInteger,
auditTest(
(value: number) => value >= 1700,
"Value must be greater than or equal to 1700",
),
)
if (errors.first_year === undefined && errors.last_year === undefined) {
if (data.first_year !== undefined && data.last_year !== undefined) {
if (data.first_year > data.last_year) {
errors.last_year =
"Last year must be greater than or equal to first year"
}
} else if (data.first_year !== undefined) {
errors.last_year = "Missing values"
} else if (data.last_year !== undefined) {
errors.first_year = "Missing values"
}
}
audit.attribute(
data,
"offset",
......@@ -385,7 +465,47 @@ export function auditWebSocketsFacetsQuery(
remainingKeys,
auditQueryOptionsSet(allFacets),
)
audit.attribute(
data,
"first_year",
true,
errors,
remainingKeys,
auditTrimString,
auditStringToNumber,
auditInteger,
auditTest(
(value: number) => value >= 1700,
"Value must be greater than or equal to 1700",
),
)
audit.attribute(data, "keyword", true, errors, remainingKeys, auditQuerySet)
audit.attribute(
data,
"last_year",
true,
errors,
remainingKeys,
auditTrimString,
auditStringToNumber,
auditInteger,
auditTest(
(value: number) => value >= 1700,
"Value must be greater than or equal to 1700",
),
)
if (errors.first_year === undefined && errors.last_year === undefined) {
if (data.first_year !== undefined && data.last_year !== undefined) {
if (data.first_year > data.last_year) {
errors.last_year =
"Last year must be greater than or equal to first year"
}
} else if (data.first_year !== undefined) {
errors.last_year = "Missing values"
} else if (data.last_year !== undefined) {
errors.first_year = "Missing values"
}
}
audit.attribute(
data,
"language",
......
......@@ -20,6 +20,7 @@
// export let minChar = 2
export let name: string | undefined = undefined
export let placeholder = ""
export let propagateEnter: boolean = false
export let required = false
export let term: string | undefined = undefined
export let type = "text"
......@@ -101,14 +102,20 @@
if (!isOpen && ignoreEnterKeyWhenMenuNotOpen) {
// Menu is closed => Do nothing and allow default form submission
} else if (itemSelected) {
event.preventDefault() // Prevent form submission.
if (!propagateEnter) {
event.preventDefault() // Prevent form submission.
}
itemSelected = false
} else if (currentItemIndex !== -1) {
event.preventDefault() // Prevent form submission.
if (!propagateEnter) {
event.preventDefault() // Prevent form submission.
}
selectItem(currentItemIndex)
} else if (forceSelectionOnEnter) {
// No item selected in list => Force selection of first item.
event.preventDefault() // Prevent form submission.
if (!propagateEnter) {
event.preventDefault() // Prevent form submission.
}
currentItemIndex = 0
selectItem(currentItemIndex)
} else {
......
......@@ -3,9 +3,10 @@
import { localize } from "../stores"
export let comparison: (a: any, b: any) => number
export let expanded: boolean
export let expandedComparison: (a: any, b: any) => number
export let items: any[]
export let shrunkenComparison: (a: any, b: any) => number
const dispatch = createEventDispatcher()
......@@ -19,7 +20,7 @@
{#if expanded && items.length > 5}
<ul class="list-inside max-h-64 overflow-y-auto text-xs text-gray-700">
{#each items as item}
{#each [...items].sort(expandedComparison) as item}
<li>
<slot {item} />
</li>
......@@ -27,7 +28,7 @@
</ul>
{:else}
<ul class="list-inside text-sm text-gray-700">
{#each [...items].sort(comparison).slice(0, 5) as item}
{#each [...items].sort(shrunkenComparison).slice(0, 5) as item}
<li>
<slot {item} />
</li>
......
<script lang="ts">
import type { element } from "svelte/internal"
import type { MdastBlockquote, MdastNode, MdastRoot } from "../markdown"
export let markdownAst: MdastRoot
export let textSize: "text-sm" | "text-xs" = "text-sm"
export let filename: String
function asMdastBlockquoteArray(nodes: MdastNode[]): MdastBlockquote[] {
return nodes as MdastBlockquote[]
}
</script>
{#each asMdastBlockquoteArray(markdownAst.children) as blockquote}
<div
id={blockquote.children[0].children[0].value}
class=" my-2 mx-0 py-8 px-16 text-left border-gray-400 rounded shadow"
>
{#each blockquote.children.slice(1, blockquote.children.length) as element}
{#if element.type == "heading" && element.depth == 1 && element.children[0].type == "text"}
<h class="text-2xl font-bold my-6 text-gray-400">
{element.children[0].value}</h
>
{:else if element.type == "heading" && element.depth == 2 && element.children[0].type == "text"}
<h class="text-xl font-bold my-6 text-gray-400">
{element.children[0].value}</h
>
{:else if element.type == "heading" && element.depth == 3 && element.children[0].type == "text"}
<h class="text-l font-bold italic my-6 text-gray-400">
{element.children[0].value}</h
>
{#each element.children as maybelink}
{#if maybelink.type == "link"}
<a class=" text-xs mx-2 text-gray-400 italic" href={maybelink.url}>
{maybelink.children[0].value}</a
>
{/if}
{/each}
{:else if element.type == "heading" && element.depth == 4 && element.children[0].type == "text"}
<h class="text-l italic my-3 text-gray-500">
{element.children[0].value}</h
>
{:else if element.type == "paragraph" && element.children[0].type == "text"}
<p class="{textSize} my-6 text-gray-500">
{element.children[0].value}
{#each element.children as maybelink}
{#if maybelink.type == "link"}
<a
class=" text-xs mx-2 text-gray-400 italic"
href={maybelink.url}
>
{maybelink.children[0].value}</a
>
{/if}
{/each}
</p>
{:else if element.type == "paragraph" && element.children[0].type == "link"}
<p class="text-xs my-4 mx-4 text-red-300 text-right">
<a href={element.children[0].url}>
{element.children[0]?.children[0]?.value}</a
>
</p>
{:else if element.type == "paragraph" && element.children[0]?.type == "image"}
<img
class="content-center align-middle justify-center text-center ml-20 my-12"
src={element.children[0].url}
alt={"imtitle"}
/>
{:else if element.type == "list"}
<ul class="{textSize} my-2 text-gray-500">
{#each element.children as listItem}
{#if listItem.children[0]?.type == "paragraph" && listItem.children[0]?.children[0]?.type == "text"}
<li class="ml-20 my-1">
-- {" " + listItem.children[0]?.children[0]?.value}
</li>
{/if}
{/each}
</ul>
{/if}
{/each}
<p class="py-0 px-20 text-center italic text-gray-300 text-opacity-50">
<a
target="_blanc"
href="https://git.nomics.world/-/ide/project/progedo/data-catalogue-textes/edit/master/-/{filename}.md"
>
Affichage automatique, fichier texte à compléter</a
>
</p>
</div>
{/each}
<script lang="ts">
import JsonTree from "svelte-json-tree"
import { getCodeBookTag } from "../accessors"
import { stores } from "@sapper/app"
import type { Study } from "../data"
import type { MdastRoot } from "../markdown"
export let markdownAst: MdastRoot
export let study: Study
const { session } = stores()
$: codeBook = study.codeBook!
</script>
<svelte:head>
<title>Study-summary | {$session.title}</title>
</svelte:head>
<!--
<p class="py-6 px-20 text-center text-2xl font-bold text-gray-400">
Dataset/ Summary
</p>-->
<div class=" flex flex-col w-auto h-auto my-6">
{#each markdownAst.children as element}
{#if element.type == "heading" && element.depth == 1 && element.children[0]?.type == "text"}
<h1 class="text-xl font-bold mt-8 text-gray-500">
{element.children[0].value.toString()}
</h1>
{:else if element.type == "heading" && element.depth == 2 && element.children[0]?.type == "text"}
<h2 class="text-xl font-bold mt-8 text-gray-500">
{element.children[0]?.value.toString()}
</h2>
{:else if element.type == "heading" && element.depth == 3 && element.children[0]?.type == "text"}
<h2 class="text-sm font-bold mt-4 text-gray-700">
{element.children[0]?.value.toString()}
</h2>
{:else if element.type == "heading" && element.depth == 4 && element.children[0]?.type == "text"}
<h3 class="text-sm font-bold italic mt-4 text-gray-700">
{element.children[0]?.value.toString()}
</h3>
{:else if element.type == "paragraph" && element.children[0]?.type == "text"}
<p class="text-sm mb-2 text-gray-700">
{element.children[0]?.value.toString()}
</p>
{:else if element.type == "paragraph" && element.children[0]?.type == "link"}
<p class="text-sm mb-2 text-gray-700 ">
{getCodeBookTag(codeBook, element.children[0]?.url)}