chore: Format using black

This commit is contained in:
PapaTutuWawa 2023-08-20 19:55:43 +02:00
parent d0ca73d8c6
commit 801d7d73d9
2 changed files with 88 additions and 65 deletions

View File

@ -13,6 +13,7 @@ import time
from optparse import OptionParser
import sys
class Config:
config: dict[str, object] = None
@ -37,6 +38,7 @@ class Config:
# Default: 1 day
return self.config.get("api_ttl", 1 * 24 * 60 * 60)
class Database:
__db = None
@ -45,15 +47,18 @@ class Database:
def initialize(self):
cur = self.__db.cursor()
cur.execute('''
cur.execute(
"""
CREATE TABLE ApiCache (
package TEXT NOT NULL PRIMARY KEY,
payload TEXT NOT NULL,
time INTEGER NOT NULL,
success INTEGER NOT NULL
)
''')
cur.execute('''
"""
)
cur.execute(
"""
CREATE TABLE PackageCache (
package TEXT NOT NULL,
version TEXT NOT NULL,
@ -61,30 +66,32 @@ class Database:
path TEXT NOT NULL,
PRIMARY KEY (package, version)
)
''')
"""
)
def get_api_cache(self, package: str):
cur = self.__db.cursor()
api = cur.execute('SELECT package, payload, time, success FROM ApiCache WHERE package = ?', (package,)).fetchone()
api = cur.execute(
"SELECT package, payload, time, success FROM ApiCache WHERE package = ?",
(package,),
).fetchone()
if api:
return ApiCacheEntry(
api[0],
api[1],
api[2],
True if api[3] == 1 else False
)
return ApiCacheEntry(api[0], api[1], api[2], True if api[3] == 1 else False)
def persist_api_cache(self, item):
cur = self.__db.cursor()
cur.execute(
'INSERT OR REPLACE INTO ApiCache VALUES (?, ?, ?, ?)',
(item.package, item.payload, item.request_time, 1 if item.success else 0)
"INSERT OR REPLACE INTO ApiCache VALUES (?, ?, ?, ?)",
(item.package, item.payload, item.request_time, 1 if item.success else 0),
)
self.__db.commit()
def get_package_cache(self, package: str, version: str):
cur = self.__db.cursor()
api = cur.execute('SELECT package, version, time, path FROM PackageCache WHERE package = ? AND version = ?', (package, version)).fetchone()
api = cur.execute(
"SELECT package, version, time, path FROM PackageCache WHERE package = ? AND version = ?",
(package, version),
).fetchone()
if api:
return PackageCacheEntry(
api[0],
@ -96,32 +103,36 @@ class Database:
def persist_package_cache(self, item):
cur = self.__db.cursor()
cur.execute(
'INSERT INTO PackageCache VALUES (?, ?, ?, ?)',
(item.package, item.version, item.request_time, item.path)
"INSERT INTO PackageCache VALUES (?, ?, ?, ?)",
(item.package, item.version, item.request_time, item.path),
)
self.__db.commit()
def patch_pubdev_api_response(resp: dict[str, object], package: str, config: Config):
# Patch the latest version
if 'latest' in resp:
version = resp['latest']['version']
resp['latest']['archive_url'] = f'{config.server_url}/api/archives/{package}/{version}'
if "latest" in resp:
version = resp["latest"]["version"]
resp["latest"][
"archive_url"
] = f"{config.server_url}/api/archives/{package}/{version}"
# Patch all other versions
new_versions = []
for release in resp['versions']:
version = release['version']
release['archive_url'] = f'{config.server_url}/api/archives/{package}/{version}'
for release in resp["versions"]:
version = release["version"]
release["archive_url"] = f"{config.server_url}/api/archives/{package}/{version}"
new_versions.append(release)
resp['versions'] = new_versions
resp["versions"] = new_versions
return resp
class PackageCacheEntry:
package: str = None
version: str = None
package: str = None
version: str = None
request_time: float = None
path: str = None
path: str = None
def __init__(self, package, version, request_time, path):
self.package = package
@ -129,6 +140,7 @@ class PackageCacheEntry:
self.request_time = request_time
self.path = path
class ApiCacheEntry:
package: str = None
payload = None
@ -144,6 +156,7 @@ class ApiCacheEntry:
def is_valid(self, ttl):
return time.time() <= self.request_time + ttl
class PubApiCacheResource:
__db: Database = None
__config: Config = None
@ -154,29 +167,32 @@ class PubApiCacheResource:
@logger.catch
async def on_get(self, req, resp, package):
#breakpoint()
# breakpoint()
cache = self.__db.get_api_cache(package)
if not cache or not cache.is_valid(self.__config.api_ttl):
logger.info(f'API response for {package} not in cache...')
logger.info(f"API response for {package} not in cache...")
r = requests.get(
f'https://pub.dev/api/packages/{package}',
f"https://pub.dev/api/packages/{package}",
headers={
'Accept': 'application/vnd.pub.v2+json',
}
"Accept": "application/vnd.pub.v2+json",
},
)
if r.status_code == 200:
payload = patch_pubdev_api_response(r.json(), package, self.__config)
else:
payload = r.json()
cache = ApiCacheEntry(package, json.dumps(payload), time.time(), r.status_code == 200)
cache = ApiCacheEntry(
package, json.dumps(payload), time.time(), r.status_code == 200
)
self.__db.persist_api_cache(cache)
logger.debug(f'Payload: {cache.payload}')
logger.debug(f"Payload: {cache.payload}")
resp.append_header('Content-Type', 'application/vnd.pub.v2+json')
resp.append_header("Content-Type", "application/vnd.pub.v2+json")
resp.text = cache.payload
resp.status = falcon.HTTP_200 if cache.success else falcon.HTTP_404
class PubPackageCacheResource:
__db: Database = None
__config: Config = None
@ -187,19 +203,22 @@ class PubPackageCacheResource:
@logger.catch
async def on_get(self, req, resp, package, version):
#breakpoint()
# breakpoint()
cache = self.__db.get_package_cache(package, version)
if not cache or not os.path.exists(cache.path):
logger.info(f'{package}:{version} not in cache. Querying...')
logger.info(f"{package}:{version} not in cache. Querying...")
package_path = os.path.join(self.__config.package_path, package)
if not os.path.exists(package_path):
os.mkdir(package_path)
path = os.path.join(package_path, f'{version}.tar.gz')
with requests.get(f'https://pub.dartlang.org/packages/{package}/versions/{version}.tar.gz', stream=True) as r:
path = os.path.join(package_path, f"{version}.tar.gz")
with requests.get(
f"https://pub.dartlang.org/packages/{package}/versions/{version}.tar.gz",
stream=True,
) as r:
r.raise_for_status()
with open(path, 'wb') as f:
with open(path, "wb") as f:
for chunk in r.iter_content(chunk_size=8196):
f.write(chunk)
cache = PackageCacheEntry(
@ -211,10 +230,11 @@ class PubPackageCacheResource:
self.__db.persist_package_cache(cache)
resp.status = falcon.HTTP_200
resp.content_type = 'application/octet-stream'
resp.stream = await aiofiles.open(cache.path, 'rb')
resp.content_type = "application/octet-stream"
resp.stream = await aiofiles.open(cache.path, "rb")
resp.content_length = os.path.getsize(cache.path)
def main():
parser = OptionParser()
parser.add_option(
@ -242,17 +262,24 @@ def main():
if should_initialize:
db.initialize()
logger.info(f'API calls have a TTL of {config.api_ttl} seconds')
logger.info(f"API calls have a TTL of {config.api_ttl} seconds")
if not os.path.exists(config.package_path):
logger.info('Creating packages directory...')
logger.info("Creating packages directory...")
os.makedirs(config.package_path)
app = falcon.asgi.App()
app.add_route('/api/packages/{package}', PubApiCacheResource(db, config))
app.add_route('/api/archives/{package}/{version}', PubPackageCacheResource(db, config))
app.add_route("/api/packages/{package}", PubApiCacheResource(db, config))
app.add_route(
"/api/archives/{package}/{version}", PubPackageCacheResource(db, config)
)
uvicorn.run(app, host=config.config.get('host', '127.0.0.1'), port=config.config.get('port', 8000))
uvicorn.run(
app,
host=config.config.get("host", "127.0.0.1"),
port=config.config.get("port", 8000),
)
if __name__ == '__main__':
if __name__ == "__main__":
main()

View File

@ -1,25 +1,21 @@
from setuptools import setup, find_packages
setup(
name='pubcached',
version='0.1',
description='Caching server for pub.dev packages',
author='Alexander \"PapaTutuWawa\"',
author_email='papatutuwawa [at] polynom.me',
name="pubcached",
version="0.1",
description="Caching server for pub.dev packages",
author='Alexander "PapaTutuWawa"',
author_email="papatutuwawa [at] polynom.me",
install_requires=[
'aiofiles>=23.1.0',
'requests>=2.29.0',
'falcon>=3.1.1',
'loguru>=0.7.0',
'toml>=0.10.2',
'uvicorn>=0.20.0'
"aiofiles>=23.1.0",
"requests>=2.29.0",
"falcon>=3.1.1",
"loguru>=0.7.0",
"toml>=0.10.2",
"uvicorn>=0.20.0",
],
packages=find_packages(),
license='MIT',
license="MIT",
zip_safe=True,
entry_points={
"console_scripts": [
"pubcached = pubcached.pubcached:main"
]
}
entry_points={"console_scripts": ["pubcached = pubcached.pubcached:main"]},
)