scripts: Add scripts to help testing and administration
This commit is contained in:
Alexander Polynomdivision
parent
4f133c55d2
commit
9144676ec2
26
scripts/README.md
Normal file
26
scripts/README.md
Normal file
@ -0,0 +1,26 @@
|
|||||||
|
# Scripts
|
||||||
|
A few scripts to help with any task
|
||||||
|
|
||||||
|
## csv_vocab_to_mongo.py
|
||||||
|
Parses the vocabulary files in `$REPO_ROOT/data/vocab/`, converts the vocabulary items
|
||||||
|
into a format that fits the database model for vocabulary items (see `backend/db.md**) and
|
||||||
|
writes the data into the MongoDB database.
|
||||||
|
|
||||||
|
**NOTE**: Requires `pymongo` to be installed via pip
|
||||||
|
|
||||||
|
### Usage
|
||||||
|
`python csv_vocab_to_mongo.py <URI> <Database>`
|
||||||
|
|
||||||
|
- `URI`: The URI of the MongoDB instance
|
||||||
|
- `Database`: The name of the database in the MongoDB instance
|
||||||
|
|
||||||
|
## add_user.js
|
||||||
|
Asks the user about his data (username, password, class** and adds the user to the database.
|
||||||
|
|
||||||
|
**NOTE**: Requires `mongodb` to be installed via npm
|
||||||
|
|
||||||
|
### Usage
|
||||||
|
`node add_user.js <URI> <Database>`
|
||||||
|
|
||||||
|
- `URI`: The URI of the MongoDB instance
|
||||||
|
- `Database`: The name of the database in the MongoDB instance
|
||||||
78
scripts/add_user.js
Normal file
78
scripts/add_user.js
Normal file
@ -0,0 +1,78 @@
|
|||||||
|
const crypto = require("crypto");
|
||||||
|
const readline = require("readline");
|
||||||
|
const process = require("process");
|
||||||
|
|
||||||
|
const MongoClient = require("mongodb").MongoClient;
|
||||||
|
|
||||||
|
if (process.argv.length < 4) {
|
||||||
|
console.log("Not enough arguments");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
function log(msg) {
|
||||||
|
console.log("[*]", msg);
|
||||||
|
}
|
||||||
|
|
||||||
|
(async () => {
|
||||||
|
const client = await (new MongoClient(process.argv[2])).connect();
|
||||||
|
const db = client.db(process.argv[3]);
|
||||||
|
|
||||||
|
const rl = readline.createInterface({
|
||||||
|
input: process.stdin,
|
||||||
|
output: process.stdout,
|
||||||
|
});
|
||||||
|
|
||||||
|
rl.question("Username? > ", username => {
|
||||||
|
rl.question("Class ID? > ", classId => {
|
||||||
|
rl.question("Password? > ", password => {
|
||||||
|
rl.question("Password (repeat)? > ", async (repeat) => {
|
||||||
|
if (repeat !== password) {
|
||||||
|
console.log(`Passwords for user ${username} don't match`);
|
||||||
|
rl.close();
|
||||||
|
} else {
|
||||||
|
// Generate a salt
|
||||||
|
log("Hashing password");
|
||||||
|
const salt = crypto.randomBytes(20).toString("hex");
|
||||||
|
const hash = crypto.pbkdf2Sync(password, salt, 50000, 512, "sha512").toString("hex");
|
||||||
|
|
||||||
|
// console.log(`Username: ${username}`);
|
||||||
|
// console.log(`Salt: ${salt}`);
|
||||||
|
// console.log(`Hash: ${hash}`);
|
||||||
|
|
||||||
|
const user = {
|
||||||
|
username,
|
||||||
|
salt,
|
||||||
|
hash,
|
||||||
|
// TODO: Fix this
|
||||||
|
classId,
|
||||||
|
// END TODO
|
||||||
|
score: 0,
|
||||||
|
showWelcome: true,
|
||||||
|
uid: 1,
|
||||||
|
|
||||||
|
lastReview: {
|
||||||
|
correct: 0,
|
||||||
|
wrong: 0,
|
||||||
|
},
|
||||||
|
|
||||||
|
lastLevel: 0,
|
||||||
|
levels: [],
|
||||||
|
vocabMetadata: {},
|
||||||
|
};
|
||||||
|
|
||||||
|
log("Writing to database...");
|
||||||
|
try {
|
||||||
|
await db.collection("users").insertOne(user);
|
||||||
|
} catch (err) {
|
||||||
|
log("Error while writing to database");
|
||||||
|
log(err);
|
||||||
|
}
|
||||||
|
|
||||||
|
rl.close();
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
})();
|
||||||
147
scripts/csv_vocab_to_mongo.py
Normal file
147
scripts/csv_vocab_to_mongo.py
Normal file
@ -0,0 +1,147 @@
|
|||||||
|
import csv
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
import pymongo
|
||||||
|
|
||||||
|
# Definitions
|
||||||
|
TYPE_NOUNS = 0
|
||||||
|
TYPE_VERBS = 1
|
||||||
|
TYPE_ADJECTIVES = 2
|
||||||
|
|
||||||
|
PATH_TO_VOCAB = "../data/vocab"
|
||||||
|
|
||||||
|
def preprocess_row(row):
|
||||||
|
return row[1:]
|
||||||
|
|
||||||
|
def genus_to_datatype(gen):
|
||||||
|
if (gen == "m"): return "Maskulin"
|
||||||
|
if (gen == "w"): return "Feminin"
|
||||||
|
if (gen == "n"): return "Neutrum"
|
||||||
|
|
||||||
|
def log(msg, err=False, tabs=0):
|
||||||
|
if (not err):
|
||||||
|
print("[*] " + "\t" * tabs + msg)
|
||||||
|
else:
|
||||||
|
print("[X] " + "\t" * tabs + msg)
|
||||||
|
|
||||||
|
def dbg(msg, tabs=0):
|
||||||
|
print("[D] " + "\t" * tabs + msg)
|
||||||
|
|
||||||
|
def csv_to_vocab(filename, type, from_id):
|
||||||
|
id = from_id + 1
|
||||||
|
vocab = []
|
||||||
|
skip = 0
|
||||||
|
path = os.path.join(PATH_TO_VOCAB, filename)
|
||||||
|
dbg("Reading from {0} ({1})".format(filename, path), tabs=2)
|
||||||
|
with open(path, newline="") as csvfile:
|
||||||
|
reader = csv.reader(csvfile, delimiter=",", quotechar="\"")
|
||||||
|
for raw in reader:
|
||||||
|
skip += 1
|
||||||
|
# Skip the header lines
|
||||||
|
# if (skip < num_lines_to_skip + 1):
|
||||||
|
if (skip < 4):
|
||||||
|
continue
|
||||||
|
|
||||||
|
# The nouns are special
|
||||||
|
row = preprocess_row(raw) if type == TYPE_NOUNS else raw
|
||||||
|
|
||||||
|
grundform = row[0]
|
||||||
|
hint = ""
|
||||||
|
mnemonic = ""
|
||||||
|
|
||||||
|
latin = {
|
||||||
|
"grundform": grundform
|
||||||
|
}
|
||||||
|
|
||||||
|
# The parsing depends on the type of word we're dealing with
|
||||||
|
bedeutungen = []
|
||||||
|
if (type == TYPE_NOUNS):
|
||||||
|
# Nomen
|
||||||
|
genitiv = row[1]
|
||||||
|
genus = genus_to_datatype(row[2])
|
||||||
|
bedeutungen = [row[3]]
|
||||||
|
if (row[4] != ""):
|
||||||
|
bedeutungen.append(row[4])
|
||||||
|
if (row[5] != ""):
|
||||||
|
bedeutungen.append(row[5])
|
||||||
|
|
||||||
|
latin["genitiv"] = genitiv
|
||||||
|
latin["genus"] = genus
|
||||||
|
elif (type == TYPE_VERBS):
|
||||||
|
# Verb
|
||||||
|
praesens = row[1]
|
||||||
|
perfekt = row[2]
|
||||||
|
bedeutungen = [row[3]]
|
||||||
|
if (row[4] != ""):
|
||||||
|
bedeutungen.append(row[4])
|
||||||
|
if (row[5] != ""):
|
||||||
|
bedeutungen.append(row[5])
|
||||||
|
|
||||||
|
latin["praesens"] = praesens
|
||||||
|
latin["perfekt"] = perfekt
|
||||||
|
latin["ppp"] = ""
|
||||||
|
elif (type == TYPE_ADJECTIVES):
|
||||||
|
# Adjektiv
|
||||||
|
endung_f = row[1]
|
||||||
|
endung_n = row[2]
|
||||||
|
bedeutungen = [row[3]]
|
||||||
|
if (row[4] != ""):
|
||||||
|
bedeutungen.append(row[4])
|
||||||
|
if (row[5] != ""):
|
||||||
|
bedeutungen.append(row[5])
|
||||||
|
|
||||||
|
latin["nominativ_a"] = endung_f
|
||||||
|
latin["nominativ_b"] = endung_n
|
||||||
|
|
||||||
|
# TODO: Hints and mnemonics
|
||||||
|
vocab.append({
|
||||||
|
"id": id,
|
||||||
|
"german": bedeutungen,
|
||||||
|
"hint": hint,
|
||||||
|
"mnemonic": mnemonic,
|
||||||
|
"type": type,
|
||||||
|
"latin": latin
|
||||||
|
})
|
||||||
|
id += 1
|
||||||
|
return vocab, id
|
||||||
|
|
||||||
|
log("Lateinicus CSV to Vocabulary DB Model")
|
||||||
|
if (len(sys.argv) < 3):
|
||||||
|
log("Not enough arguments!", err=True)
|
||||||
|
log("Usage: csv_vocab_to_mongo.py <URI> <Database>", err=True)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
log("Generating vocabulary")
|
||||||
|
id = 0
|
||||||
|
vocab = []
|
||||||
|
# Nouns
|
||||||
|
log("Nouns...", tabs=1)
|
||||||
|
nouns, last_id = csv_to_vocab("Nomen.csv", TYPE_NOUNS, 0)
|
||||||
|
vocab += nouns
|
||||||
|
id = last_id
|
||||||
|
|
||||||
|
log("Verbs...", tabs=1)
|
||||||
|
verbs, last_id = csv_to_vocab("Verben.csv", TYPE_VERBS, last_id)
|
||||||
|
vocab += verbs
|
||||||
|
id = last_id
|
||||||
|
|
||||||
|
log("Adjectives...", tabs=1)
|
||||||
|
adj, last_id = csv_to_vocab("Adjektive.csv", TYPE_ADJECTIVES, last_id)
|
||||||
|
vocab += adj
|
||||||
|
id = last_id
|
||||||
|
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
# Connect to the database
|
||||||
|
log("Inserting vocabulary into database")
|
||||||
|
log("Connecting...", tabs=1)
|
||||||
|
client = pymongo.MongoClient(sys.argv[1])
|
||||||
|
log("Getting DB...", tabs=1)
|
||||||
|
db = client[sys.argv[2]]
|
||||||
|
log("Inserting...", tabs=1)
|
||||||
|
res = db["vocabulary"].insert_many(vocab)
|
||||||
|
|
||||||
|
if (len(res.inserted_ids) != len(vocab)):
|
||||||
|
log("Not enough elements were added to the database", err=True, tabs=1)
|
||||||
|
else:
|
||||||
|
log("Success", tabs=1)
|
||||||
79
scripts/package-lock.json
generated
Normal file
79
scripts/package-lock.json
generated
Normal file
@ -0,0 +1,79 @@
|
|||||||
|
{
|
||||||
|
"requires": true,
|
||||||
|
"lockfileVersion": 1,
|
||||||
|
"dependencies": {
|
||||||
|
"bson": {
|
||||||
|
"version": "1.1.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/bson/-/bson-1.1.0.tgz",
|
||||||
|
"integrity": "sha512-9Aeai9TacfNtWXOYarkFJRW2CWo+dRon+fuLZYJmvLV3+MiUp0bEI6IAZfXEIg7/Pl/7IWlLaDnhzTsD81etQA=="
|
||||||
|
},
|
||||||
|
"memory-pager": {
|
||||||
|
"version": "1.1.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.1.0.tgz",
|
||||||
|
"integrity": "sha512-Mf9OHV/Y7h6YWDxTzX/b4ZZ4oh9NSXblQL8dtPCOomOtZciEHxePR78+uHFLLlsk01A6jVHhHsQZZ/WcIPpnzg==",
|
||||||
|
"optional": true
|
||||||
|
},
|
||||||
|
"mongodb": {
|
||||||
|
"version": "3.1.6",
|
||||||
|
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.1.6.tgz",
|
||||||
|
"integrity": "sha512-E5QJuXQoMlT7KyCYqNNMfAkhfQD79AT4F8Xd+6x37OX+8BL17GyXyWvfm6wuyx4wnzCCPoCSLeMeUN2S7dU9yw==",
|
||||||
|
"requires": {
|
||||||
|
"mongodb-core": "3.1.5",
|
||||||
|
"safe-buffer": "^5.1.2"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"mongodb-core": {
|
||||||
|
"version": "3.1.5",
|
||||||
|
"resolved": "https://registry.npmjs.org/mongodb-core/-/mongodb-core-3.1.5.tgz",
|
||||||
|
"integrity": "sha512-emT/tM4ZBinqd6RZok+EzDdtN4LjYJIckv71qQVOEFmvXgT5cperZegVmTgox/1cx4XQu6LJ5ZuIwipP/eKdQg==",
|
||||||
|
"requires": {
|
||||||
|
"bson": "^1.1.0",
|
||||||
|
"require_optional": "^1.0.1",
|
||||||
|
"safe-buffer": "^5.1.2",
|
||||||
|
"saslprep": "^1.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"require_optional": {
|
||||||
|
"version": "1.0.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/require_optional/-/require_optional-1.0.1.tgz",
|
||||||
|
"integrity": "sha512-qhM/y57enGWHAe3v/NcwML6a3/vfESLe/sGM2dII+gEO0BpKRUkWZow/tyloNqJyN6kXSl3RyyM8Ll5D/sJP8g==",
|
||||||
|
"requires": {
|
||||||
|
"resolve-from": "^2.0.0",
|
||||||
|
"semver": "^5.1.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"resolve-from": {
|
||||||
|
"version": "2.0.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-2.0.0.tgz",
|
||||||
|
"integrity": "sha1-lICrIOlP+h2egKgEx+oUdhGWa1c="
|
||||||
|
},
|
||||||
|
"safe-buffer": {
|
||||||
|
"version": "5.1.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
|
||||||
|
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
|
||||||
|
},
|
||||||
|
"saslprep": {
|
||||||
|
"version": "1.0.2",
|
||||||
|
"resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.2.tgz",
|
||||||
|
"integrity": "sha512-4cDsYuAjXssUSjxHKRe4DTZC0agDwsCqcMqtJAQPzC74nJ7LfAJflAtC1Zed5hMzEQKj82d3tuzqdGNRsLJ4Gw==",
|
||||||
|
"optional": true,
|
||||||
|
"requires": {
|
||||||
|
"sparse-bitfield": "^3.0.3"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"semver": {
|
||||||
|
"version": "5.5.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/semver/-/semver-5.5.1.tgz",
|
||||||
|
"integrity": "sha512-PqpAxfrEhlSUWge8dwIp4tZnQ25DIOthpiaHNIthsjEFQD6EvqUKUDM7L8O2rShkFccYo1VjJR0coWfNkCubRw=="
|
||||||
|
},
|
||||||
|
"sparse-bitfield": {
|
||||||
|
"version": "3.0.3",
|
||||||
|
"resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",
|
||||||
|
"integrity": "sha1-/0rm5oZWBWuks+eSqzM004JzyhE=",
|
||||||
|
"optional": true,
|
||||||
|
"requires": {
|
||||||
|
"memory-pager": "^1.0.2"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user