From 9144676ec299956e997896c7811d5ee9a26e9fda Mon Sep 17 00:00:00 2001 From: Alexander Polynomdivision Date: Mon, 1 Oct 2018 17:08:37 +0200 Subject: [PATCH] scripts: Add scripts to help testing and administration --- scripts/README.md | 26 ++++++ scripts/add_user.js | 78 ++++++++++++++++++ scripts/csv_vocab_to_mongo.py | 147 ++++++++++++++++++++++++++++++++++ scripts/package-lock.json | 79 ++++++++++++++++++ 4 files changed, 330 insertions(+) create mode 100644 scripts/README.md create mode 100644 scripts/add_user.js create mode 100644 scripts/csv_vocab_to_mongo.py create mode 100644 scripts/package-lock.json diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..85ed35e --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,26 @@ +# Scripts +A few scripts to help with any task + +## csv_vocab_to_mongo.py +Parses the vocabulary files in `$REPO_ROOT/data/vocab/`, converts the vocabulary items +into a format that fits the database model for vocabulary items (see `backend/db.md**) and +writes the data into the MongoDB database. + +**NOTE**: Requires `pymongo` to be installed via pip + +### Usage +`python csv_vocab_to_mongo.py ` + +- `URI`: The URI of the MongoDB instance +- `Database`: The name of the database in the MongoDB instance + +## add_user.js +Asks the user about his data (username, password, class** and adds the user to the database. + +**NOTE**: Requires `mongodb` to be installed via npm + +### Usage +`node add_user.js ` + +- `URI`: The URI of the MongoDB instance +- `Database`: The name of the database in the MongoDB instance diff --git a/scripts/add_user.js b/scripts/add_user.js new file mode 100644 index 0000000..43b5499 --- /dev/null +++ b/scripts/add_user.js @@ -0,0 +1,78 @@ +const crypto = require("crypto"); +const readline = require("readline"); +const process = require("process"); + +const MongoClient = require("mongodb").MongoClient; + +if (process.argv.length < 4) { + console.log("Not enough arguments"); + process.exit(1); +} + +function log(msg) { + console.log("[*]", msg); +} + +(async () => { + const client = await (new MongoClient(process.argv[2])).connect(); + const db = client.db(process.argv[3]); + + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); + + rl.question("Username? > ", username => { + rl.question("Class ID? > ", classId => { + rl.question("Password? > ", password => { + rl.question("Password (repeat)? > ", async (repeat) => { + if (repeat !== password) { + console.log(`Passwords for user ${username} don't match`); + rl.close(); + } else { + // Generate a salt + log("Hashing password"); + const salt = crypto.randomBytes(20).toString("hex"); + const hash = crypto.pbkdf2Sync(password, salt, 50000, 512, "sha512").toString("hex"); + + // console.log(`Username: ${username}`); + // console.log(`Salt: ${salt}`); + // console.log(`Hash: ${hash}`); + + const user = { + username, + salt, + hash, + // TODO: Fix this + classId, + // END TODO + score: 0, + showWelcome: true, + uid: 1, + + lastReview: { + correct: 0, + wrong: 0, + }, + + lastLevel: 0, + levels: [], + vocabMetadata: {}, + }; + + log("Writing to database..."); + try { + await db.collection("users").insertOne(user); + } catch (err) { + log("Error while writing to database"); + log(err); + } + + rl.close(); + process.exit(0); + } + }); + }); + }); + }); +})(); diff --git a/scripts/csv_vocab_to_mongo.py b/scripts/csv_vocab_to_mongo.py new file mode 100644 index 0000000..53aa526 --- /dev/null +++ b/scripts/csv_vocab_to_mongo.py @@ -0,0 +1,147 @@ +import csv +import sys +import os +import pymongo + +# Definitions +TYPE_NOUNS = 0 +TYPE_VERBS = 1 +TYPE_ADJECTIVES = 2 + +PATH_TO_VOCAB = "../data/vocab" + +def preprocess_row(row): + return row[1:] + +def genus_to_datatype(gen): + if (gen == "m"): return "Maskulin" + if (gen == "w"): return "Feminin" + if (gen == "n"): return "Neutrum" + +def log(msg, err=False, tabs=0): + if (not err): + print("[*] " + "\t" * tabs + msg) + else: + print("[X] " + "\t" * tabs + msg) + +def dbg(msg, tabs=0): + print("[D] " + "\t" * tabs + msg) + +def csv_to_vocab(filename, type, from_id): + id = from_id + 1 + vocab = [] + skip = 0 + path = os.path.join(PATH_TO_VOCAB, filename) + dbg("Reading from {0} ({1})".format(filename, path), tabs=2) + with open(path, newline="") as csvfile: + reader = csv.reader(csvfile, delimiter=",", quotechar="\"") + for raw in reader: + skip += 1 + # Skip the header lines + # if (skip < num_lines_to_skip + 1): + if (skip < 4): + continue + + # The nouns are special + row = preprocess_row(raw) if type == TYPE_NOUNS else raw + + grundform = row[0] + hint = "" + mnemonic = "" + + latin = { + "grundform": grundform + } + + # The parsing depends on the type of word we're dealing with + bedeutungen = [] + if (type == TYPE_NOUNS): + # Nomen + genitiv = row[1] + genus = genus_to_datatype(row[2]) + bedeutungen = [row[3]] + if (row[4] != ""): + bedeutungen.append(row[4]) + if (row[5] != ""): + bedeutungen.append(row[5]) + + latin["genitiv"] = genitiv + latin["genus"] = genus + elif (type == TYPE_VERBS): + # Verb + praesens = row[1] + perfekt = row[2] + bedeutungen = [row[3]] + if (row[4] != ""): + bedeutungen.append(row[4]) + if (row[5] != ""): + bedeutungen.append(row[5]) + + latin["praesens"] = praesens + latin["perfekt"] = perfekt + latin["ppp"] = "" + elif (type == TYPE_ADJECTIVES): + # Adjektiv + endung_f = row[1] + endung_n = row[2] + bedeutungen = [row[3]] + if (row[4] != ""): + bedeutungen.append(row[4]) + if (row[5] != ""): + bedeutungen.append(row[5]) + + latin["nominativ_a"] = endung_f + latin["nominativ_b"] = endung_n + + # TODO: Hints and mnemonics + vocab.append({ + "id": id, + "german": bedeutungen, + "hint": hint, + "mnemonic": mnemonic, + "type": type, + "latin": latin + }) + id += 1 + return vocab, id + +log("Lateinicus CSV to Vocabulary DB Model") +if (len(sys.argv) < 3): + log("Not enough arguments!", err=True) + log("Usage: csv_vocab_to_mongo.py ", err=True) + sys.exit(1) + +log("Generating vocabulary") +id = 0 +vocab = [] +# Nouns +log("Nouns...", tabs=1) +nouns, last_id = csv_to_vocab("Nomen.csv", TYPE_NOUNS, 0) +vocab += nouns +id = last_id + +log("Verbs...", tabs=1) +verbs, last_id = csv_to_vocab("Verben.csv", TYPE_VERBS, last_id) +vocab += verbs +id = last_id + +log("Adjectives...", tabs=1) +adj, last_id = csv_to_vocab("Adjektive.csv", TYPE_ADJECTIVES, last_id) +vocab += adj +id = last_id + +sys.exit(1) + +# Connect to the database +log("Inserting vocabulary into database") +log("Connecting...", tabs=1) +client = pymongo.MongoClient(sys.argv[1]) +log("Getting DB...", tabs=1) +db = client[sys.argv[2]] +log("Inserting...", tabs=1) +res = db["vocabulary"].insert_many(vocab) + +if (len(res.inserted_ids) != len(vocab)): + log("Not enough elements were added to the database", err=True, tabs=1) +else: + log("Success", tabs=1) diff --git a/scripts/package-lock.json b/scripts/package-lock.json new file mode 100644 index 0000000..c329e9e --- /dev/null +++ b/scripts/package-lock.json @@ -0,0 +1,79 @@ +{ + "requires": true, + "lockfileVersion": 1, + "dependencies": { + "bson": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/bson/-/bson-1.1.0.tgz", + "integrity": "sha512-9Aeai9TacfNtWXOYarkFJRW2CWo+dRon+fuLZYJmvLV3+MiUp0bEI6IAZfXEIg7/Pl/7IWlLaDnhzTsD81etQA==" + }, + "memory-pager": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.1.0.tgz", + "integrity": "sha512-Mf9OHV/Y7h6YWDxTzX/b4ZZ4oh9NSXblQL8dtPCOomOtZciEHxePR78+uHFLLlsk01A6jVHhHsQZZ/WcIPpnzg==", + "optional": true + }, + "mongodb": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.1.6.tgz", + "integrity": "sha512-E5QJuXQoMlT7KyCYqNNMfAkhfQD79AT4F8Xd+6x37OX+8BL17GyXyWvfm6wuyx4wnzCCPoCSLeMeUN2S7dU9yw==", + "requires": { + "mongodb-core": "3.1.5", + "safe-buffer": "^5.1.2" + } + }, + "mongodb-core": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/mongodb-core/-/mongodb-core-3.1.5.tgz", + "integrity": "sha512-emT/tM4ZBinqd6RZok+EzDdtN4LjYJIckv71qQVOEFmvXgT5cperZegVmTgox/1cx4XQu6LJ5ZuIwipP/eKdQg==", + "requires": { + "bson": "^1.1.0", + "require_optional": "^1.0.1", + "safe-buffer": "^5.1.2", + "saslprep": "^1.0.0" + } + }, + "require_optional": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/require_optional/-/require_optional-1.0.1.tgz", + "integrity": "sha512-qhM/y57enGWHAe3v/NcwML6a3/vfESLe/sGM2dII+gEO0BpKRUkWZow/tyloNqJyN6kXSl3RyyM8Ll5D/sJP8g==", + "requires": { + "resolve-from": "^2.0.0", + "semver": "^5.1.0" + } + }, + "resolve-from": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-2.0.0.tgz", + "integrity": "sha1-lICrIOlP+h2egKgEx+oUdhGWa1c=" + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "saslprep": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.2.tgz", + "integrity": "sha512-4cDsYuAjXssUSjxHKRe4DTZC0agDwsCqcMqtJAQPzC74nJ7LfAJflAtC1Zed5hMzEQKj82d3tuzqdGNRsLJ4Gw==", + "optional": true, + "requires": { + "sparse-bitfield": "^3.0.3" + } + }, + "semver": { + "version": "5.5.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.5.1.tgz", + "integrity": "sha512-PqpAxfrEhlSUWge8dwIp4tZnQ25DIOthpiaHNIthsjEFQD6EvqUKUDM7L8O2rShkFccYo1VjJR0coWfNkCubRw==" + }, + "sparse-bitfield": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", + "integrity": "sha1-/0rm5oZWBWuks+eSqzM004JzyhE=", + "optional": true, + "requires": { + "memory-pager": "^1.0.2" + } + } + } +}