From 7b1706b2ddf69a612ae49579853d3fdf3171cde9 Mon Sep 17 00:00:00 2001 From: Tanishq Dubey Date: Fri, 12 Aug 2022 15:23:42 -0400 Subject: [PATCH] Initial Commit --- .gitignore | 160 +++++++++++++++++++++++++++++++++++++++++ Dockerfile | 10 +++ config/localkube.yaml | 5 ++ main.py | 129 +++++++++++++++++++++++++++++++++ requirements.txt | 5 ++ templates/browser.html | 31 ++++++++ templates/index.html | 25 +++++++ 7 files changed, 365 insertions(+) create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 config/localkube.yaml create mode 100644 main.py create mode 100644 requirements.txt create mode 100644 templates/browser.html create mode 100644 templates/index.html diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..68bc17f --- /dev/null +++ b/.gitignore @@ -0,0 +1,160 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e57ea71 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM docker.co.clearstreet.io/clst/clst-python:3.9.5-slim-buster + +WORKDIR /usr/src/app + +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +CMD [ "python", "./main.py" ] diff --git a/config/localkube.yaml b/config/localkube.yaml new file mode 100644 index 0000000..d9bc7ba --- /dev/null +++ b/config/localkube.yaml @@ -0,0 +1,5 @@ +--- +aws: + access_key: fake + secret_key: fake + endpoint_url: http://aws:4566 diff --git a/main.py b/main.py new file mode 100644 index 0000000..8585f11 --- /dev/null +++ b/main.py @@ -0,0 +1,129 @@ +import os + +import boto3 + +from flask import Flask, send_file, render_template, redirect +from gestalt import Gestalt + +from clearstreet.logger import create_logger + + +def sanitize_key(key): + return key.replace("/", "-") + + +def rebuild_key(key): + return key.replace("-", "/") + + +def get_path(key): + s = key.split("/") + return "/".join(s[:len(s) - 1]) + + +def get_filename(key): + s = key.split("/") + return s[len(s) - 1] + + +def build_s3_client(config: Gestalt): + session = boto3.Session() + access = config.get_string("aws.access_key") + secret = config.get_string("aws.secret_key") + url = config.get_string("aws.endpoint_url", 's3.amazonaws.com') + s3 = session.client( + service_name="s3", + aws_access_key_id=access, + aws_secret_access_key=secret, + endpoint_url=url, + ) + return s3 + + +def list_buckets(client): + ret = [] + list = client.list_buckets()['Buckets'] + for bucket in list: + ret.append(bucket["Name"]) + ret.sort() + return ret + + +def list_in_bucket(client, bucket, prefix): + ret = [] + items = [] + + if prefix is None or len(prefix) == 0: + items = client.list_objects_v2(Bucket=bucket, Delimiter="/") + else: + items = client.list_objects_v2(Bucket=bucket, Prefix=f"{prefix}/", Delimiter="/") + + for item in items.get('Contents', []): + if item.get('Key', None) is not None: + ret.append(item['Key']) + ret.sort() + + dirs = [] + for item in items.get('CommonPrefixes', []): + if item.get('Prefix', None) is not None: + dirs.append(item['Prefix']) + + dirs.sort() + return ret, dirs + + +def download_object(client, bucket_name, key): + key = rebuild_key(key) + filename = "/tmp" + get_filename(key) + client.download_file(Bucket=bucket_name, Key=key, Filename=filename) + return filename + + +env = os.environ.get('ENV', 'localkube') + +logger = create_logger("q1") + +app = Flask(__name__) + +logger.info("loading configurations", env=env) +g = Gestalt() +g.add_config_file(f'./config/{env}.yaml') +g.build_config() +g.auto_env() + +s3client = build_s3_client(g) + + +@app.route('/download//') +def download(bucket, path): + global s3client + fname = download_object(s3client, bucket, path) + return send_file(fname, as_attachment=True) + + +@app.route('/browse/', defaults={'path': ''}) +@app.route('/browse//', defaults={'path': ''}) +@app.route('/browse//') +@app.route('/browse///') +def withinBucket(bucket, path): + global s3client + logger.info("inside bucket", bucket=bucket, path=path) + items, dirs = list_in_bucket(s3client, bucket, path) + return render_template('browser.html', items=items, dirs=dirs, bucket=bucket, path=path) + + +@app.route('/browse') +@app.route('/browse/') +def sendhome(): + return redirect("/", code=302) + + +@app.route('/') +def root(): + global s3client + buckets = list_buckets(s3client) + return render_template('index.html', buckets=buckets) + + +if __name__ == "__main__": + app.run(host="0.0.0.0", port=8081, debug=True) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..3c001b6 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +Flask==2.1.2 +clearstreet.logger==2.0.2 +boto3==1.24.47 +botocore==1.27.48 +gestalt-cfg==3.0.0 diff --git a/templates/browser.html b/templates/browser.html new file mode 100644 index 0000000..2262a15 --- /dev/null +++ b/templates/browser.html @@ -0,0 +1,31 @@ + + + + + + + + + + + + Q1 + + +

Q1: A Simple S3 Browser

+

Directories

+
    + {% for item in dirs %} +
  • {{item}}
    • + {% endfor %} +
+

Files

+
    + {% for item in items %} +
  • {{item}} -- Download
    • + {% endfor %} +
+ Back + + + diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..af555a7 --- /dev/null +++ b/templates/index.html @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + Q1 + + +

Q1: A Simple S3 Browser

+
    + {% for bucket in buckets %} +
  • {{bucket}}
    • + {% endfor %} +
+ + +