scripts: Add scripts to help testing and administration
This commit is contained in:
Alexander Polynomdivision
parent
4f133c55d2
commit
9144676ec2
26
scripts/README.md
Normal file
26
scripts/README.md
Normal file
@ -0,0 +1,26 @@
|
||||
# Scripts
|
||||
A few scripts to help with any task
|
||||
|
||||
## csv_vocab_to_mongo.py
|
||||
Parses the vocabulary files in `$REPO_ROOT/data/vocab/`, converts the vocabulary items
|
||||
into a format that fits the database model for vocabulary items (see `backend/db.md**) and
|
||||
writes the data into the MongoDB database.
|
||||
|
||||
**NOTE**: Requires `pymongo` to be installed via pip
|
||||
|
||||
### Usage
|
||||
`python csv_vocab_to_mongo.py <URI> <Database>`
|
||||
|
||||
- `URI`: The URI of the MongoDB instance
|
||||
- `Database`: The name of the database in the MongoDB instance
|
||||
|
||||
## add_user.js
|
||||
Asks the user about his data (username, password, class** and adds the user to the database.
|
||||
|
||||
**NOTE**: Requires `mongodb` to be installed via npm
|
||||
|
||||
### Usage
|
||||
`node add_user.js <URI> <Database>`
|
||||
|
||||
- `URI`: The URI of the MongoDB instance
|
||||
- `Database`: The name of the database in the MongoDB instance
|
||||
78
scripts/add_user.js
Normal file
78
scripts/add_user.js
Normal file
@ -0,0 +1,78 @@
|
||||
const crypto = require("crypto");
|
||||
const readline = require("readline");
|
||||
const process = require("process");
|
||||
|
||||
const MongoClient = require("mongodb").MongoClient;
|
||||
|
||||
if (process.argv.length < 4) {
|
||||
console.log("Not enough arguments");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
function log(msg) {
|
||||
console.log("[*]", msg);
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const client = await (new MongoClient(process.argv[2])).connect();
|
||||
const db = client.db(process.argv[3]);
|
||||
|
||||
const rl = readline.createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout,
|
||||
});
|
||||
|
||||
rl.question("Username? > ", username => {
|
||||
rl.question("Class ID? > ", classId => {
|
||||
rl.question("Password? > ", password => {
|
||||
rl.question("Password (repeat)? > ", async (repeat) => {
|
||||
if (repeat !== password) {
|
||||
console.log(`Passwords for user ${username} don't match`);
|
||||
rl.close();
|
||||
} else {
|
||||
// Generate a salt
|
||||
log("Hashing password");
|
||||
const salt = crypto.randomBytes(20).toString("hex");
|
||||
const hash = crypto.pbkdf2Sync(password, salt, 50000, 512, "sha512").toString("hex");
|
||||
|
||||
// console.log(`Username: ${username}`);
|
||||
// console.log(`Salt: ${salt}`);
|
||||
// console.log(`Hash: ${hash}`);
|
||||
|
||||
const user = {
|
||||
username,
|
||||
salt,
|
||||
hash,
|
||||
// TODO: Fix this
|
||||
classId,
|
||||
// END TODO
|
||||
score: 0,
|
||||
showWelcome: true,
|
||||
uid: 1,
|
||||
|
||||
lastReview: {
|
||||
correct: 0,
|
||||
wrong: 0,
|
||||
},
|
||||
|
||||
lastLevel: 0,
|
||||
levels: [],
|
||||
vocabMetadata: {},
|
||||
};
|
||||
|
||||
log("Writing to database...");
|
||||
try {
|
||||
await db.collection("users").insertOne(user);
|
||||
} catch (err) {
|
||||
log("Error while writing to database");
|
||||
log(err);
|
||||
}
|
||||
|
||||
rl.close();
|
||||
process.exit(0);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
})();
|
||||
147
scripts/csv_vocab_to_mongo.py
Normal file
147
scripts/csv_vocab_to_mongo.py
Normal file
@ -0,0 +1,147 @@
|
||||
import csv
|
||||
import sys
|
||||
import os
|
||||
import pymongo
|
||||
|
||||
# Definitions
|
||||
TYPE_NOUNS = 0
|
||||
TYPE_VERBS = 1
|
||||
TYPE_ADJECTIVES = 2
|
||||
|
||||
PATH_TO_VOCAB = "../data/vocab"
|
||||
|
||||
def preprocess_row(row):
|
||||
return row[1:]
|
||||
|
||||
def genus_to_datatype(gen):
|
||||
if (gen == "m"): return "Maskulin"
|
||||
if (gen == "w"): return "Feminin"
|
||||
if (gen == "n"): return "Neutrum"
|
||||
|
||||
def log(msg, err=False, tabs=0):
|
||||
if (not err):
|
||||
print("[*] " + "\t" * tabs + msg)
|
||||
else:
|
||||
print("[X] " + "\t" * tabs + msg)
|
||||
|
||||
def dbg(msg, tabs=0):
|
||||
print("[D] " + "\t" * tabs + msg)
|
||||
|
||||
def csv_to_vocab(filename, type, from_id):
|
||||
id = from_id + 1
|
||||
vocab = []
|
||||
skip = 0
|
||||
path = os.path.join(PATH_TO_VOCAB, filename)
|
||||
dbg("Reading from {0} ({1})".format(filename, path), tabs=2)
|
||||
with open(path, newline="") as csvfile:
|
||||
reader = csv.reader(csvfile, delimiter=",", quotechar="\"")
|
||||
for raw in reader:
|
||||
skip += 1
|
||||
# Skip the header lines
|
||||
# if (skip < num_lines_to_skip + 1):
|
||||
if (skip < 4):
|
||||
continue
|
||||
|
||||
# The nouns are special
|
||||
row = preprocess_row(raw) if type == TYPE_NOUNS else raw
|
||||
|
||||
grundform = row[0]
|
||||
hint = ""
|
||||
mnemonic = ""
|
||||
|
||||
latin = {
|
||||
"grundform": grundform
|
||||
}
|
||||
|
||||
# The parsing depends on the type of word we're dealing with
|
||||
bedeutungen = []
|
||||
if (type == TYPE_NOUNS):
|
||||
# Nomen
|
||||
genitiv = row[1]
|
||||
genus = genus_to_datatype(row[2])
|
||||
bedeutungen = [row[3]]
|
||||
if (row[4] != ""):
|
||||
bedeutungen.append(row[4])
|
||||
if (row[5] != ""):
|
||||
bedeutungen.append(row[5])
|
||||
|
||||
latin["genitiv"] = genitiv
|
||||
latin["genus"] = genus
|
||||
elif (type == TYPE_VERBS):
|
||||
# Verb
|
||||
praesens = row[1]
|
||||
perfekt = row[2]
|
||||
bedeutungen = [row[3]]
|
||||
if (row[4] != ""):
|
||||
bedeutungen.append(row[4])
|
||||
if (row[5] != ""):
|
||||
bedeutungen.append(row[5])
|
||||
|
||||
latin["praesens"] = praesens
|
||||
latin["perfekt"] = perfekt
|
||||
latin["ppp"] = ""
|
||||
elif (type == TYPE_ADJECTIVES):
|
||||
# Adjektiv
|
||||
endung_f = row[1]
|
||||
endung_n = row[2]
|
||||
bedeutungen = [row[3]]
|
||||
if (row[4] != ""):
|
||||
bedeutungen.append(row[4])
|
||||
if (row[5] != ""):
|
||||
bedeutungen.append(row[5])
|
||||
|
||||
latin["nominativ_a"] = endung_f
|
||||
latin["nominativ_b"] = endung_n
|
||||
|
||||
# TODO: Hints and mnemonics
|
||||
vocab.append({
|
||||
"id": id,
|
||||
"german": bedeutungen,
|
||||
"hint": hint,
|
||||
"mnemonic": mnemonic,
|
||||
"type": type,
|
||||
"latin": latin
|
||||
})
|
||||
id += 1
|
||||
return vocab, id
|
||||
|
||||
log("Lateinicus CSV to Vocabulary DB Model")
|
||||
if (len(sys.argv) < 3):
|
||||
log("Not enough arguments!", err=True)
|
||||
log("Usage: csv_vocab_to_mongo.py <URI> <Database>", err=True)
|
||||
sys.exit(1)
|
||||
|
||||
log("Generating vocabulary")
|
||||
id = 0
|
||||
vocab = []
|
||||
# Nouns
|
||||
log("Nouns...", tabs=1)
|
||||
nouns, last_id = csv_to_vocab("Nomen.csv", TYPE_NOUNS, 0)
|
||||
vocab += nouns
|
||||
id = last_id
|
||||
|
||||
log("Verbs...", tabs=1)
|
||||
verbs, last_id = csv_to_vocab("Verben.csv", TYPE_VERBS, last_id)
|
||||
vocab += verbs
|
||||
id = last_id
|
||||
|
||||
log("Adjectives...", tabs=1)
|
||||
adj, last_id = csv_to_vocab("Adjektive.csv", TYPE_ADJECTIVES, last_id)
|
||||
vocab += adj
|
||||
id = last_id
|
||||
|
||||
sys.exit(1)
|
||||
|
||||
# Connect to the database
|
||||
log("Inserting vocabulary into database")
|
||||
log("Connecting...", tabs=1)
|
||||
client = pymongo.MongoClient(sys.argv[1])
|
||||
log("Getting DB...", tabs=1)
|
||||
db = client[sys.argv[2]]
|
||||
log("Inserting...", tabs=1)
|
||||
res = db["vocabulary"].insert_many(vocab)
|
||||
|
||||
if (len(res.inserted_ids) != len(vocab)):
|
||||
log("Not enough elements were added to the database", err=True, tabs=1)
|
||||
else:
|
||||
log("Success", tabs=1)
|
||||
79
scripts/package-lock.json
generated
Normal file
79
scripts/package-lock.json
generated
Normal file
@ -0,0 +1,79 @@
|
||||
{
|
||||
"requires": true,
|
||||
"lockfileVersion": 1,
|
||||
"dependencies": {
|
||||
"bson": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/bson/-/bson-1.1.0.tgz",
|
||||
"integrity": "sha512-9Aeai9TacfNtWXOYarkFJRW2CWo+dRon+fuLZYJmvLV3+MiUp0bEI6IAZfXEIg7/Pl/7IWlLaDnhzTsD81etQA=="
|
||||
},
|
||||
"memory-pager": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.1.0.tgz",
|
||||
"integrity": "sha512-Mf9OHV/Y7h6YWDxTzX/b4ZZ4oh9NSXblQL8dtPCOomOtZciEHxePR78+uHFLLlsk01A6jVHhHsQZZ/WcIPpnzg==",
|
||||
"optional": true
|
||||
},
|
||||
"mongodb": {
|
||||
"version": "3.1.6",
|
||||
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.1.6.tgz",
|
||||
"integrity": "sha512-E5QJuXQoMlT7KyCYqNNMfAkhfQD79AT4F8Xd+6x37OX+8BL17GyXyWvfm6wuyx4wnzCCPoCSLeMeUN2S7dU9yw==",
|
||||
"requires": {
|
||||
"mongodb-core": "3.1.5",
|
||||
"safe-buffer": "^5.1.2"
|
||||
}
|
||||
},
|
||||
"mongodb-core": {
|
||||
"version": "3.1.5",
|
||||
"resolved": "https://registry.npmjs.org/mongodb-core/-/mongodb-core-3.1.5.tgz",
|
||||
"integrity": "sha512-emT/tM4ZBinqd6RZok+EzDdtN4LjYJIckv71qQVOEFmvXgT5cperZegVmTgox/1cx4XQu6LJ5ZuIwipP/eKdQg==",
|
||||
"requires": {
|
||||
"bson": "^1.1.0",
|
||||
"require_optional": "^1.0.1",
|
||||
"safe-buffer": "^5.1.2",
|
||||
"saslprep": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"require_optional": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/require_optional/-/require_optional-1.0.1.tgz",
|
||||
"integrity": "sha512-qhM/y57enGWHAe3v/NcwML6a3/vfESLe/sGM2dII+gEO0BpKRUkWZow/tyloNqJyN6kXSl3RyyM8Ll5D/sJP8g==",
|
||||
"requires": {
|
||||
"resolve-from": "^2.0.0",
|
||||
"semver": "^5.1.0"
|
||||
}
|
||||
},
|
||||
"resolve-from": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-2.0.0.tgz",
|
||||
"integrity": "sha1-lICrIOlP+h2egKgEx+oUdhGWa1c="
|
||||
},
|
||||
"safe-buffer": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
|
||||
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
|
||||
},
|
||||
"saslprep": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.2.tgz",
|
||||
"integrity": "sha512-4cDsYuAjXssUSjxHKRe4DTZC0agDwsCqcMqtJAQPzC74nJ7LfAJflAtC1Zed5hMzEQKj82d3tuzqdGNRsLJ4Gw==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"sparse-bitfield": "^3.0.3"
|
||||
}
|
||||
},
|
||||
"semver": {
|
||||
"version": "5.5.1",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-5.5.1.tgz",
|
||||
"integrity": "sha512-PqpAxfrEhlSUWge8dwIp4tZnQ25DIOthpiaHNIthsjEFQD6EvqUKUDM7L8O2rShkFccYo1VjJR0coWfNkCubRw=="
|
||||
},
|
||||
"sparse-bitfield": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",
|
||||
"integrity": "sha1-/0rm5oZWBWuks+eSqzM004JzyhE=",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"memory-pager": "^1.0.2"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user