diff --git a/pubcached/pubcached.py b/pubcached/pubcached.py index df029e5..40a5584 100644 --- a/pubcached/pubcached.py +++ b/pubcached/pubcached.py @@ -13,6 +13,7 @@ import time from optparse import OptionParser import sys + class Config: config: dict[str, object] = None @@ -37,6 +38,7 @@ class Config: # Default: 1 day return self.config.get("api_ttl", 1 * 24 * 60 * 60) + class Database: __db = None @@ -45,15 +47,18 @@ class Database: def initialize(self): cur = self.__db.cursor() - cur.execute(''' + cur.execute( + """ CREATE TABLE ApiCache ( package TEXT NOT NULL PRIMARY KEY, payload TEXT NOT NULL, time INTEGER NOT NULL, success INTEGER NOT NULL ) - ''') - cur.execute(''' + """ + ) + cur.execute( + """ CREATE TABLE PackageCache ( package TEXT NOT NULL, version TEXT NOT NULL, @@ -61,30 +66,32 @@ class Database: path TEXT NOT NULL, PRIMARY KEY (package, version) ) - ''') + """ + ) def get_api_cache(self, package: str): cur = self.__db.cursor() - api = cur.execute('SELECT package, payload, time, success FROM ApiCache WHERE package = ?', (package,)).fetchone() + api = cur.execute( + "SELECT package, payload, time, success FROM ApiCache WHERE package = ?", + (package,), + ).fetchone() if api: - return ApiCacheEntry( - api[0], - api[1], - api[2], - True if api[3] == 1 else False - ) + return ApiCacheEntry(api[0], api[1], api[2], True if api[3] == 1 else False) def persist_api_cache(self, item): cur = self.__db.cursor() cur.execute( - 'INSERT OR REPLACE INTO ApiCache VALUES (?, ?, ?, ?)', - (item.package, item.payload, item.request_time, 1 if item.success else 0) + "INSERT OR REPLACE INTO ApiCache VALUES (?, ?, ?, ?)", + (item.package, item.payload, item.request_time, 1 if item.success else 0), ) self.__db.commit() def get_package_cache(self, package: str, version: str): cur = self.__db.cursor() - api = cur.execute('SELECT package, version, time, path FROM PackageCache WHERE package = ? AND version = ?', (package, version)).fetchone() + api = cur.execute( + "SELECT package, version, time, path FROM PackageCache WHERE package = ? AND version = ?", + (package, version), + ).fetchone() if api: return PackageCacheEntry( api[0], @@ -96,32 +103,36 @@ class Database: def persist_package_cache(self, item): cur = self.__db.cursor() cur.execute( - 'INSERT INTO PackageCache VALUES (?, ?, ?, ?)', - (item.package, item.version, item.request_time, item.path) + "INSERT INTO PackageCache VALUES (?, ?, ?, ?)", + (item.package, item.version, item.request_time, item.path), ) self.__db.commit() + def patch_pubdev_api_response(resp: dict[str, object], package: str, config: Config): # Patch the latest version - if 'latest' in resp: - version = resp['latest']['version'] - resp['latest']['archive_url'] = f'{config.server_url}/api/archives/{package}/{version}' + if "latest" in resp: + version = resp["latest"]["version"] + resp["latest"][ + "archive_url" + ] = f"{config.server_url}/api/archives/{package}/{version}" # Patch all other versions new_versions = [] - for release in resp['versions']: - version = release['version'] - release['archive_url'] = f'{config.server_url}/api/archives/{package}/{version}' + for release in resp["versions"]: + version = release["version"] + release["archive_url"] = f"{config.server_url}/api/archives/{package}/{version}" new_versions.append(release) - resp['versions'] = new_versions + resp["versions"] = new_versions return resp + class PackageCacheEntry: - package: str = None - version: str = None + package: str = None + version: str = None request_time: float = None - path: str = None + path: str = None def __init__(self, package, version, request_time, path): self.package = package @@ -129,6 +140,7 @@ class PackageCacheEntry: self.request_time = request_time self.path = path + class ApiCacheEntry: package: str = None payload = None @@ -144,6 +156,7 @@ class ApiCacheEntry: def is_valid(self, ttl): return time.time() <= self.request_time + ttl + class PubApiCacheResource: __db: Database = None __config: Config = None @@ -154,29 +167,32 @@ class PubApiCacheResource: @logger.catch async def on_get(self, req, resp, package): - #breakpoint() - cache = self.__db.get_api_cache(package) + # breakpoint() + cache = self.__db.get_api_cache(package) if not cache or not cache.is_valid(self.__config.api_ttl): - logger.info(f'API response for {package} not in cache...') + logger.info(f"API response for {package} not in cache...") r = requests.get( - f'https://pub.dev/api/packages/{package}', + f"https://pub.dev/api/packages/{package}", headers={ - 'Accept': 'application/vnd.pub.v2+json', - } + "Accept": "application/vnd.pub.v2+json", + }, ) if r.status_code == 200: payload = patch_pubdev_api_response(r.json(), package, self.__config) else: payload = r.json() - cache = ApiCacheEntry(package, json.dumps(payload), time.time(), r.status_code == 200) + cache = ApiCacheEntry( + package, json.dumps(payload), time.time(), r.status_code == 200 + ) self.__db.persist_api_cache(cache) - logger.debug(f'Payload: {cache.payload}') + logger.debug(f"Payload: {cache.payload}") - resp.append_header('Content-Type', 'application/vnd.pub.v2+json') + resp.append_header("Content-Type", "application/vnd.pub.v2+json") resp.text = cache.payload resp.status = falcon.HTTP_200 if cache.success else falcon.HTTP_404 + class PubPackageCacheResource: __db: Database = None __config: Config = None @@ -187,19 +203,22 @@ class PubPackageCacheResource: @logger.catch async def on_get(self, req, resp, package, version): - #breakpoint() + # breakpoint() cache = self.__db.get_package_cache(package, version) if not cache or not os.path.exists(cache.path): - logger.info(f'{package}:{version} not in cache. Querying...') + logger.info(f"{package}:{version} not in cache. Querying...") package_path = os.path.join(self.__config.package_path, package) if not os.path.exists(package_path): os.mkdir(package_path) - path = os.path.join(package_path, f'{version}.tar.gz') - with requests.get(f'https://pub.dartlang.org/packages/{package}/versions/{version}.tar.gz', stream=True) as r: + path = os.path.join(package_path, f"{version}.tar.gz") + with requests.get( + f"https://pub.dartlang.org/packages/{package}/versions/{version}.tar.gz", + stream=True, + ) as r: r.raise_for_status() - with open(path, 'wb') as f: + with open(path, "wb") as f: for chunk in r.iter_content(chunk_size=8196): f.write(chunk) cache = PackageCacheEntry( @@ -211,10 +230,11 @@ class PubPackageCacheResource: self.__db.persist_package_cache(cache) resp.status = falcon.HTTP_200 - resp.content_type = 'application/octet-stream' - resp.stream = await aiofiles.open(cache.path, 'rb') + resp.content_type = "application/octet-stream" + resp.stream = await aiofiles.open(cache.path, "rb") resp.content_length = os.path.getsize(cache.path) + def main(): parser = OptionParser() parser.add_option( @@ -242,17 +262,24 @@ def main(): if should_initialize: db.initialize() - logger.info(f'API calls have a TTL of {config.api_ttl} seconds') + logger.info(f"API calls have a TTL of {config.api_ttl} seconds") if not os.path.exists(config.package_path): - logger.info('Creating packages directory...') + logger.info("Creating packages directory...") os.makedirs(config.package_path) app = falcon.asgi.App() - app.add_route('/api/packages/{package}', PubApiCacheResource(db, config)) - app.add_route('/api/archives/{package}/{version}', PubPackageCacheResource(db, config)) + app.add_route("/api/packages/{package}", PubApiCacheResource(db, config)) + app.add_route( + "/api/archives/{package}/{version}", PubPackageCacheResource(db, config) + ) - uvicorn.run(app, host=config.config.get('host', '127.0.0.1'), port=config.config.get('port', 8000)) + uvicorn.run( + app, + host=config.config.get("host", "127.0.0.1"), + port=config.config.get("port", 8000), + ) -if __name__ == '__main__': + +if __name__ == "__main__": main() diff --git a/setup.py b/setup.py index ce56d4c..8c3f1b1 100644 --- a/setup.py +++ b/setup.py @@ -1,25 +1,21 @@ from setuptools import setup, find_packages setup( - name='pubcached', - version='0.1', - description='Caching server for pub.dev packages', - author='Alexander \"PapaTutuWawa\"', - author_email='papatutuwawa [at] polynom.me', + name="pubcached", + version="0.1", + description="Caching server for pub.dev packages", + author='Alexander "PapaTutuWawa"', + author_email="papatutuwawa [at] polynom.me", install_requires=[ - 'aiofiles>=23.1.0', - 'requests>=2.29.0', - 'falcon>=3.1.1', - 'loguru>=0.7.0', - 'toml>=0.10.2', - 'uvicorn>=0.20.0' + "aiofiles>=23.1.0", + "requests>=2.29.0", + "falcon>=3.1.1", + "loguru>=0.7.0", + "toml>=0.10.2", + "uvicorn>=0.20.0", ], packages=find_packages(), - license='MIT', + license="MIT", zip_safe=True, - entry_points={ - "console_scripts": [ - "pubcached = pubcached.pubcached:main" - ] - } + entry_points={"console_scripts": ["pubcached = pubcached.pubcached:main"]}, )