Compare commits
No commits in common. "275a415248fe87ff4d2ad57331bed2267e6d108c" and "7b1706b2ddf69a612ae49579853d3fdf3171cde9" have entirely different histories.
275a415248
...
7b1706b2dd
@ -1,4 +1,4 @@
|
||||
FROM python:3
|
||||
FROM docker.co.clearstreet.io/clst/clst-python:3.9.5-slim-buster
|
||||
|
||||
WORKDIR /usr/src/app
|
||||
|
||||
|
||||
70
main.py
70
main.py
@ -1,9 +1,12 @@
|
||||
import os
|
||||
|
||||
import boto3
|
||||
from flask import Flask, redirect, render_template, send_file
|
||||
|
||||
from flask import Flask, send_file, render_template, redirect
|
||||
from gestalt import Gestalt
|
||||
|
||||
from clearstreet.logger import create_logger
|
||||
|
||||
|
||||
def sanitize_key(key):
|
||||
return key.replace("/", "-")
|
||||
@ -15,7 +18,7 @@ def rebuild_key(key):
|
||||
|
||||
def get_path(key):
|
||||
s = key.split("/")
|
||||
return "/".join(s[: len(s) - 1])
|
||||
return "/".join(s[:len(s) - 1])
|
||||
|
||||
|
||||
def get_filename(key):
|
||||
@ -27,7 +30,7 @@ def build_s3_client(config: Gestalt):
|
||||
session = boto3.Session()
|
||||
access = config.get_string("aws.access_key")
|
||||
secret = config.get_string("aws.secret_key")
|
||||
url = config.get_string("aws.endpoint_url", "s3.amazonaws.com")
|
||||
url = config.get_string("aws.endpoint_url", 's3.amazonaws.com')
|
||||
s3 = session.client(
|
||||
service_name="s3",
|
||||
aws_access_key_id=access,
|
||||
@ -39,7 +42,7 @@ def build_s3_client(config: Gestalt):
|
||||
|
||||
def list_buckets(client):
|
||||
ret = []
|
||||
list = client.list_buckets()["Buckets"]
|
||||
list = client.list_buckets()['Buckets']
|
||||
for bucket in list:
|
||||
ret.append(bucket["Name"])
|
||||
ret.sort()
|
||||
@ -53,19 +56,17 @@ def list_in_bucket(client, bucket, prefix):
|
||||
if prefix is None or len(prefix) == 0:
|
||||
items = client.list_objects_v2(Bucket=bucket, Delimiter="/")
|
||||
else:
|
||||
items = client.list_objects_v2(
|
||||
Bucket=bucket, Prefix=f"{prefix}/", Delimiter="/"
|
||||
)
|
||||
items = client.list_objects_v2(Bucket=bucket, Prefix=f"{prefix}/", Delimiter="/")
|
||||
|
||||
for item in items.get("Contents", []):
|
||||
if item.get("Key", None) is not None:
|
||||
ret.append(item["Key"])
|
||||
for item in items.get('Contents', []):
|
||||
if item.get('Key', None) is not None:
|
||||
ret.append(item['Key'])
|
||||
ret.sort()
|
||||
|
||||
dirs = []
|
||||
for item in items.get("CommonPrefixes", []):
|
||||
if item.get("Prefix", None) is not None:
|
||||
dirs.append(item["Prefix"])
|
||||
for item in items.get('CommonPrefixes', []):
|
||||
if item.get('Prefix', None) is not None:
|
||||
dirs.append(item['Prefix'])
|
||||
|
||||
dirs.sort()
|
||||
return ret, dirs
|
||||
@ -78,59 +79,50 @@ def download_object(client, bucket_name, key):
|
||||
return filename
|
||||
|
||||
|
||||
env = os.environ.get("ENV", "localkube")
|
||||
env = os.environ.get('ENV', 'localkube')
|
||||
|
||||
logger = create_logger("q1")
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
logger.info("loading configurations", env=env)
|
||||
g = Gestalt()
|
||||
g.add_config_file(f"./config/{env}.yaml")
|
||||
g.add_config_file(f'./config/{env}.yaml')
|
||||
g.build_config()
|
||||
g.auto_env()
|
||||
|
||||
s3client = build_s3_client(g)
|
||||
|
||||
|
||||
@app.route("/download/<bucket>/<path:path>")
|
||||
@app.route('/download/<bucket>/<path:path>')
|
||||
def download(bucket, path):
|
||||
global s3client
|
||||
fname = download_object(s3client, bucket, path)
|
||||
return send_file(fname, as_attachment=True)
|
||||
|
||||
|
||||
@app.route("/browse/<bucket>", defaults={"path": ""})
|
||||
@app.route("/browse/<bucket>/", defaults={"path": ""})
|
||||
@app.route("/browse/<bucket>/<path:path>")
|
||||
@app.route("/browse/<bucket>/<path:path>/")
|
||||
@app.route('/browse/<bucket>', defaults={'path': ''})
|
||||
@app.route('/browse/<bucket>/', defaults={'path': ''})
|
||||
@app.route('/browse/<bucket>/<path:path>')
|
||||
@app.route('/browse/<bucket>/<path:path>/')
|
||||
def withinBucket(bucket, path):
|
||||
global s3client
|
||||
logger.info("inside bucket", bucket=bucket, path=path)
|
||||
items, dirs = list_in_bucket(s3client, bucket, path)
|
||||
item_names = []
|
||||
dir_names = []
|
||||
for item in items:
|
||||
item_names.append(item.split("/")[-1])
|
||||
for item in dirs:
|
||||
dir_names.append(item.split("/")[-2])
|
||||
return render_template(
|
||||
"browser.html",
|
||||
items=items,
|
||||
dirs=dirs,
|
||||
bucket=bucket,
|
||||
path=path,
|
||||
itemnames=item_names,
|
||||
dirnames=dir_names,
|
||||
)
|
||||
return render_template('browser.html', items=items, dirs=dirs, bucket=bucket, path=path)
|
||||
|
||||
|
||||
@app.route("/browse")
|
||||
@app.route("/browse/")
|
||||
@app.route('/browse')
|
||||
@app.route('/browse/')
|
||||
def sendhome():
|
||||
return redirect("/", code=302)
|
||||
|
||||
|
||||
@app.route("/")
|
||||
@app.route('/')
|
||||
def root():
|
||||
global s3client
|
||||
buckets = list_buckets(s3client)
|
||||
return render_template("index.html", buckets=buckets)
|
||||
return render_template('index.html', buckets=buckets)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
Flask==2.1.2
|
||||
clearstreet.logger==2.0.2
|
||||
boto3==1.24.47
|
||||
botocore==1.27.48
|
||||
gestalt-cfg==3.0.0
|
||||
|
||||
@ -2,35 +2,30 @@
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<!--Import Google Icon Font-->
|
||||
<link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet">
|
||||
<!--Import materialize.css-->
|
||||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/materialize/1.0.0/css/materialize.min.css">
|
||||
|
||||
<!-- Latest compiled and minified CSS -->
|
||||
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap@3.4.1/dist/css/bootstrap.min.css" integrity="sha384-HSMxcRTRxnN+Bdg0JdbxYKrThecOKuH5zCYotlSAcp1+c8xmyTe9GYg1l9a69psu" crossorigin="anonymous">
|
||||
|
||||
<!-- Optional theme -->
|
||||
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap@3.4.1/dist/css/bootstrap-theme.min.css" integrity="sha384-6pzBo3FDv/PJ8r2KRkGHifhEocL+1X2rVCTTkUfGk7/0pbek5mMa1upzvWbrUbOZ" crossorigin="anonymous">
|
||||
|
||||
<!-- Latest compiled and minified JavaScript -->
|
||||
<script src="https://cdn.jsdelivr.net/npm/bootstrap@3.4.1/dist/js/bootstrap.min.js" integrity="sha384-aJ21OjlMXNL5UyIl/XNwTMqvzeRMZH2w8c5cRVpzpU8Y5bApTppSuUkhZXN0VxHd" crossorigin="anonymous"></script>
|
||||
|
||||
<!--Let browser know website is optimized for mobile-->
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
|
||||
<title>Q1</title>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<h1>Q1: A Simple S3 Browser</h1>
|
||||
<h2>{{bucket}}/{{path}}</h2>
|
||||
<h3>Directories</h3>
|
||||
<ul>
|
||||
{% for item in range(dirs|length) %}
|
||||
<li><a href="/browse/{{bucket}}/{{dirs[item]}}">{{dirnames[item]}}</a></li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
<h3>Files</h3>
|
||||
<ul>
|
||||
{% for item in range(items|length) %}
|
||||
<li>{{itemnames[item]}} -- <a href="/download/{{bucket}}/{{items[item]}}">Download</a></li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
<a href="../"><b>Back</b></a>
|
||||
</div>
|
||||
<h1>Q1: A Simple S3 Browser</h1>
|
||||
<h3>Directories</h3>
|
||||
<ul>
|
||||
{% for item in dirs %}
|
||||
<li><a href="/browse/{{bucket}}/{{item}}">{{item}}</a></li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
<h3>Files</h3>
|
||||
<ul>
|
||||
{% for item in items %}
|
||||
<li>{{item}} -- <a href="/download/{{bucket}}/{{item}}">Download</a></li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
<a href="../"><b>Back</b></a>
|
||||
<script type="text/javascript" src="js/materialize.min.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@ -2,25 +2,24 @@
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<!-- Latest compiled and minified CSS -->
|
||||
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap@3.4.1/dist/css/bootstrap.min.css" integrity="sha384-HSMxcRTRxnN+Bdg0JdbxYKrThecOKuH5zCYotlSAcp1+c8xmyTe9GYg1l9a69psu" crossorigin="anonymous">
|
||||
|
||||
<!-- Optional theme -->
|
||||
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap@3.4.1/dist/css/bootstrap-theme.min.css" integrity="sha384-6pzBo3FDv/PJ8r2KRkGHifhEocL+1X2rVCTTkUfGk7/0pbek5mMa1upzvWbrUbOZ" crossorigin="anonymous">
|
||||
<!--Import Google Icon Font-->
|
||||
<link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet">
|
||||
<!--Import materialize.css-->
|
||||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/materialize/1.0.0/css/materialize.min.css">
|
||||
|
||||
<!-- Latest compiled and minified JavaScript -->
|
||||
<script src="https://cdn.jsdelivr.net/npm/bootstrap@3.4.1/dist/js/bootstrap.min.js" integrity="sha384-aJ21OjlMXNL5UyIl/XNwTMqvzeRMZH2w8c5cRVpzpU8Y5bApTppSuUkhZXN0VxHd" crossorigin="anonymous"></script>
|
||||
<!--Let browser know website is optimized for mobile-->
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
|
||||
|
||||
<title>Q1</title>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<h1>Q1: A Simple S3 Browser</h1>
|
||||
<ul>
|
||||
{% for bucket in buckets %}
|
||||
<li><a href="/browse/{{bucket}}">{{bucket}}</a></li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</div>
|
||||
<h1>Q1: A Simple S3 Browser</h1>
|
||||
<ul>
|
||||
{% for bucket in buckets %}
|
||||
<li><a href="/browse/{{bucket}}">{{bucket}}</a></li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
<script type="text/javascript" src="js/materialize.min.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user