Initial commit

This commit is contained in:
PapaTutuWawa 2023-08-20 19:54:52 +02:00
commit d0ca73d8c6
6 changed files with 320 additions and 0 deletions

8
.gitignore vendored Normal file
View File

@ -0,0 +1,8 @@
__pycache__/
build/
config.toml
*.sqlite
dist/
*.egg-info
tmp/
venv/

22
LICENSE Normal file
View File

@ -0,0 +1,22 @@
MIT License
Copyright (c) 2023 Alexander "PapaTutuWawa"
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

7
README.md Normal file
View File

@ -0,0 +1,7 @@
# pubcached
A caching front-end for the pub.dev package repository.
## License
See `./LICENSE`.

0
pubcached/__init__.py Normal file
View File

258
pubcached/pubcached.py Normal file
View File

@ -0,0 +1,258 @@
import falcon
import falcon.asgi
import requests
from loguru import logger
import uvicorn
import aiofiles
import toml
import json
import os
import sqlite3
import time
from optparse import OptionParser
import sys
class Config:
config: dict[str, object] = None
# Construct the config from the file at @path
def __init__(self, path):
self.config = toml.load(path)
@property
def db_path(self):
return self.config["db_path"]
@property
def server_url(self):
return self.config.get("server_url", "127.0.0.1:8000")
@property
def package_path(self):
return self.config["package_path"]
@property
def api_ttl(self):
# Default: 1 day
return self.config.get("api_ttl", 1 * 24 * 60 * 60)
class Database:
__db = None
def __init__(self, path: str):
self.__db = sqlite3.connect(path)
def initialize(self):
cur = self.__db.cursor()
cur.execute('''
CREATE TABLE ApiCache (
package TEXT NOT NULL PRIMARY KEY,
payload TEXT NOT NULL,
time INTEGER NOT NULL,
success INTEGER NOT NULL
)
''')
cur.execute('''
CREATE TABLE PackageCache (
package TEXT NOT NULL,
version TEXT NOT NULL,
time INTEGER NOT NULL,
path TEXT NOT NULL,
PRIMARY KEY (package, version)
)
''')
def get_api_cache(self, package: str):
cur = self.__db.cursor()
api = cur.execute('SELECT package, payload, time, success FROM ApiCache WHERE package = ?', (package,)).fetchone()
if api:
return ApiCacheEntry(
api[0],
api[1],
api[2],
True if api[3] == 1 else False
)
def persist_api_cache(self, item):
cur = self.__db.cursor()
cur.execute(
'INSERT OR REPLACE INTO ApiCache VALUES (?, ?, ?, ?)',
(item.package, item.payload, item.request_time, 1 if item.success else 0)
)
self.__db.commit()
def get_package_cache(self, package: str, version: str):
cur = self.__db.cursor()
api = cur.execute('SELECT package, version, time, path FROM PackageCache WHERE package = ? AND version = ?', (package, version)).fetchone()
if api:
return PackageCacheEntry(
api[0],
api[1],
api[2],
api[3],
)
def persist_package_cache(self, item):
cur = self.__db.cursor()
cur.execute(
'INSERT INTO PackageCache VALUES (?, ?, ?, ?)',
(item.package, item.version, item.request_time, item.path)
)
self.__db.commit()
def patch_pubdev_api_response(resp: dict[str, object], package: str, config: Config):
# Patch the latest version
if 'latest' in resp:
version = resp['latest']['version']
resp['latest']['archive_url'] = f'{config.server_url}/api/archives/{package}/{version}'
# Patch all other versions
new_versions = []
for release in resp['versions']:
version = release['version']
release['archive_url'] = f'{config.server_url}/api/archives/{package}/{version}'
new_versions.append(release)
resp['versions'] = new_versions
return resp
class PackageCacheEntry:
package: str = None
version: str = None
request_time: float = None
path: str = None
def __init__(self, package, version, request_time, path):
self.package = package
self.version = version
self.request_time = request_time
self.path = path
class ApiCacheEntry:
package: str = None
payload = None
request_time = None
success = None
def __init__(self, package, payload, request_time, success):
self.package = package
self.payload = payload
self.request_time = request_time
self.success = success
def is_valid(self, ttl):
return time.time() <= self.request_time + ttl
class PubApiCacheResource:
__db: Database = None
__config: Config = None
def __init__(self, db: Database, config: Config):
self.__db = db
self.__config = config
@logger.catch
async def on_get(self, req, resp, package):
#breakpoint()
cache = self.__db.get_api_cache(package)
if not cache or not cache.is_valid(self.__config.api_ttl):
logger.info(f'API response for {package} not in cache...')
r = requests.get(
f'https://pub.dev/api/packages/{package}',
headers={
'Accept': 'application/vnd.pub.v2+json',
}
)
if r.status_code == 200:
payload = patch_pubdev_api_response(r.json(), package, self.__config)
else:
payload = r.json()
cache = ApiCacheEntry(package, json.dumps(payload), time.time(), r.status_code == 200)
self.__db.persist_api_cache(cache)
logger.debug(f'Payload: {cache.payload}')
resp.append_header('Content-Type', 'application/vnd.pub.v2+json')
resp.text = cache.payload
resp.status = falcon.HTTP_200 if cache.success else falcon.HTTP_404
class PubPackageCacheResource:
__db: Database = None
__config: Config = None
def __init__(self, db: Database, config: Config):
self.__db = db
self.__config = config
@logger.catch
async def on_get(self, req, resp, package, version):
#breakpoint()
cache = self.__db.get_package_cache(package, version)
if not cache or not os.path.exists(cache.path):
logger.info(f'{package}:{version} not in cache. Querying...')
package_path = os.path.join(self.__config.package_path, package)
if not os.path.exists(package_path):
os.mkdir(package_path)
path = os.path.join(package_path, f'{version}.tar.gz')
with requests.get(f'https://pub.dartlang.org/packages/{package}/versions/{version}.tar.gz', stream=True) as r:
r.raise_for_status()
with open(path, 'wb') as f:
for chunk in r.iter_content(chunk_size=8196):
f.write(chunk)
cache = PackageCacheEntry(
package,
version,
time.time(),
path,
)
self.__db.persist_package_cache(cache)
resp.status = falcon.HTTP_200
resp.content_type = 'application/octet-stream'
resp.stream = await aiofiles.open(cache.path, 'rb')
resp.content_length = os.path.getsize(cache.path)
def main():
parser = OptionParser()
parser.add_option(
"-d", "--debug", dest="debug", help="Enable debug logging", action="store_true"
)
parser.add_option(
"-c",
"--config",
dest="config",
help="Location of the config.toml",
default="/etc/pubcached/config.toml",
)
(options, args) = parser.parse_args()
# Configure verbosity
logger.remove()
logger.add(
sys.stdout,
level="DEBUG" if options.debug else "INFO",
)
config = Config(options.config)
should_initialize = not os.path.exists(config.db_path)
db = Database(config.db_path)
if should_initialize:
db.initialize()
logger.info(f'API calls have a TTL of {config.api_ttl} seconds')
if not os.path.exists(config.package_path):
logger.info('Creating packages directory...')
os.makedirs(config.package_path)
app = falcon.asgi.App()
app.add_route('/api/packages/{package}', PubApiCacheResource(db, config))
app.add_route('/api/archives/{package}/{version}', PubPackageCacheResource(db, config))
uvicorn.run(app, host=config.config.get('host', '127.0.0.1'), port=config.config.get('port', 8000))
if __name__ == '__main__':
main()

25
setup.py Normal file
View File

@ -0,0 +1,25 @@
from setuptools import setup, find_packages
setup(
name='pubcached',
version='0.1',
description='Caching server for pub.dev packages',
author='Alexander \"PapaTutuWawa\"',
author_email='papatutuwawa [at] polynom.me',
install_requires=[
'aiofiles>=23.1.0',
'requests>=2.29.0',
'falcon>=3.1.1',
'loguru>=0.7.0',
'toml>=0.10.2',
'uvicorn>=0.20.0'
],
packages=find_packages(),
license='MIT',
zip_safe=True,
entry_points={
"console_scripts": [
"pubcached = pubcached.pubcached:main"
]
}
)