Compare commits
2 Commits
7b1706b2dd
...
275a415248
Author | SHA1 | Date | |
---|---|---|---|
|
275a415248 | ||
|
c953336669 |
@ -1,4 +1,4 @@
|
|||||||
FROM docker.co.clearstreet.io/clst/clst-python:3.9.5-slim-buster
|
FROM python:3
|
||||||
|
|
||||||
WORKDIR /usr/src/app
|
WORKDIR /usr/src/app
|
||||||
|
|
||||||
|
68
main.py
68
main.py
@ -1,12 +1,9 @@
|
|||||||
import os
|
import os
|
||||||
|
|
||||||
import boto3
|
import boto3
|
||||||
|
from flask import Flask, redirect, render_template, send_file
|
||||||
from flask import Flask, send_file, render_template, redirect
|
|
||||||
from gestalt import Gestalt
|
from gestalt import Gestalt
|
||||||
|
|
||||||
from clearstreet.logger import create_logger
|
|
||||||
|
|
||||||
|
|
||||||
def sanitize_key(key):
|
def sanitize_key(key):
|
||||||
return key.replace("/", "-")
|
return key.replace("/", "-")
|
||||||
@ -30,7 +27,7 @@ def build_s3_client(config: Gestalt):
|
|||||||
session = boto3.Session()
|
session = boto3.Session()
|
||||||
access = config.get_string("aws.access_key")
|
access = config.get_string("aws.access_key")
|
||||||
secret = config.get_string("aws.secret_key")
|
secret = config.get_string("aws.secret_key")
|
||||||
url = config.get_string("aws.endpoint_url", 's3.amazonaws.com')
|
url = config.get_string("aws.endpoint_url", "s3.amazonaws.com")
|
||||||
s3 = session.client(
|
s3 = session.client(
|
||||||
service_name="s3",
|
service_name="s3",
|
||||||
aws_access_key_id=access,
|
aws_access_key_id=access,
|
||||||
@ -42,7 +39,7 @@ def build_s3_client(config: Gestalt):
|
|||||||
|
|
||||||
def list_buckets(client):
|
def list_buckets(client):
|
||||||
ret = []
|
ret = []
|
||||||
list = client.list_buckets()['Buckets']
|
list = client.list_buckets()["Buckets"]
|
||||||
for bucket in list:
|
for bucket in list:
|
||||||
ret.append(bucket["Name"])
|
ret.append(bucket["Name"])
|
||||||
ret.sort()
|
ret.sort()
|
||||||
@ -56,17 +53,19 @@ def list_in_bucket(client, bucket, prefix):
|
|||||||
if prefix is None or len(prefix) == 0:
|
if prefix is None or len(prefix) == 0:
|
||||||
items = client.list_objects_v2(Bucket=bucket, Delimiter="/")
|
items = client.list_objects_v2(Bucket=bucket, Delimiter="/")
|
||||||
else:
|
else:
|
||||||
items = client.list_objects_v2(Bucket=bucket, Prefix=f"{prefix}/", Delimiter="/")
|
items = client.list_objects_v2(
|
||||||
|
Bucket=bucket, Prefix=f"{prefix}/", Delimiter="/"
|
||||||
|
)
|
||||||
|
|
||||||
for item in items.get('Contents', []):
|
for item in items.get("Contents", []):
|
||||||
if item.get('Key', None) is not None:
|
if item.get("Key", None) is not None:
|
||||||
ret.append(item['Key'])
|
ret.append(item["Key"])
|
||||||
ret.sort()
|
ret.sort()
|
||||||
|
|
||||||
dirs = []
|
dirs = []
|
||||||
for item in items.get('CommonPrefixes', []):
|
for item in items.get("CommonPrefixes", []):
|
||||||
if item.get('Prefix', None) is not None:
|
if item.get("Prefix", None) is not None:
|
||||||
dirs.append(item['Prefix'])
|
dirs.append(item["Prefix"])
|
||||||
|
|
||||||
dirs.sort()
|
dirs.sort()
|
||||||
return ret, dirs
|
return ret, dirs
|
||||||
@ -79,50 +78,59 @@ def download_object(client, bucket_name, key):
|
|||||||
return filename
|
return filename
|
||||||
|
|
||||||
|
|
||||||
env = os.environ.get('ENV', 'localkube')
|
env = os.environ.get("ENV", "localkube")
|
||||||
|
|
||||||
logger = create_logger("q1")
|
|
||||||
|
|
||||||
app = Flask(__name__)
|
app = Flask(__name__)
|
||||||
|
|
||||||
logger.info("loading configurations", env=env)
|
|
||||||
g = Gestalt()
|
g = Gestalt()
|
||||||
g.add_config_file(f'./config/{env}.yaml')
|
g.add_config_file(f"./config/{env}.yaml")
|
||||||
g.build_config()
|
g.build_config()
|
||||||
g.auto_env()
|
g.auto_env()
|
||||||
|
|
||||||
s3client = build_s3_client(g)
|
s3client = build_s3_client(g)
|
||||||
|
|
||||||
|
|
||||||
@app.route('/download/<bucket>/<path:path>')
|
@app.route("/download/<bucket>/<path:path>")
|
||||||
def download(bucket, path):
|
def download(bucket, path):
|
||||||
global s3client
|
global s3client
|
||||||
fname = download_object(s3client, bucket, path)
|
fname = download_object(s3client, bucket, path)
|
||||||
return send_file(fname, as_attachment=True)
|
return send_file(fname, as_attachment=True)
|
||||||
|
|
||||||
|
|
||||||
@app.route('/browse/<bucket>', defaults={'path': ''})
|
@app.route("/browse/<bucket>", defaults={"path": ""})
|
||||||
@app.route('/browse/<bucket>/', defaults={'path': ''})
|
@app.route("/browse/<bucket>/", defaults={"path": ""})
|
||||||
@app.route('/browse/<bucket>/<path:path>')
|
@app.route("/browse/<bucket>/<path:path>")
|
||||||
@app.route('/browse/<bucket>/<path:path>/')
|
@app.route("/browse/<bucket>/<path:path>/")
|
||||||
def withinBucket(bucket, path):
|
def withinBucket(bucket, path):
|
||||||
global s3client
|
global s3client
|
||||||
logger.info("inside bucket", bucket=bucket, path=path)
|
|
||||||
items, dirs = list_in_bucket(s3client, bucket, path)
|
items, dirs = list_in_bucket(s3client, bucket, path)
|
||||||
return render_template('browser.html', items=items, dirs=dirs, bucket=bucket, path=path)
|
item_names = []
|
||||||
|
dir_names = []
|
||||||
|
for item in items:
|
||||||
|
item_names.append(item.split("/")[-1])
|
||||||
|
for item in dirs:
|
||||||
|
dir_names.append(item.split("/")[-2])
|
||||||
|
return render_template(
|
||||||
|
"browser.html",
|
||||||
|
items=items,
|
||||||
|
dirs=dirs,
|
||||||
|
bucket=bucket,
|
||||||
|
path=path,
|
||||||
|
itemnames=item_names,
|
||||||
|
dirnames=dir_names,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@app.route('/browse')
|
@app.route("/browse")
|
||||||
@app.route('/browse/')
|
@app.route("/browse/")
|
||||||
def sendhome():
|
def sendhome():
|
||||||
return redirect("/", code=302)
|
return redirect("/", code=302)
|
||||||
|
|
||||||
|
|
||||||
@app.route('/')
|
@app.route("/")
|
||||||
def root():
|
def root():
|
||||||
global s3client
|
global s3client
|
||||||
buckets = list_buckets(s3client)
|
buckets = list_buckets(s3client)
|
||||||
return render_template('index.html', buckets=buckets)
|
return render_template("index.html", buckets=buckets)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
@ -1,5 +1,4 @@
|
|||||||
Flask==2.1.2
|
Flask==2.1.2
|
||||||
clearstreet.logger==2.0.2
|
|
||||||
boto3==1.24.47
|
boto3==1.24.47
|
||||||
botocore==1.27.48
|
botocore==1.27.48
|
||||||
gestalt-cfg==3.0.0
|
gestalt-cfg==3.0.0
|
||||||
|
@ -2,30 +2,35 @@
|
|||||||
<html lang="en">
|
<html lang="en">
|
||||||
<head>
|
<head>
|
||||||
<meta charset="UTF-8">
|
<meta charset="UTF-8">
|
||||||
<!--Import Google Icon Font-->
|
|
||||||
<link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet">
|
|
||||||
<!--Import materialize.css-->
|
|
||||||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/materialize/1.0.0/css/materialize.min.css">
|
|
||||||
|
|
||||||
<!--Let browser know website is optimized for mobile-->
|
<!-- Latest compiled and minified CSS -->
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
|
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap@3.4.1/dist/css/bootstrap.min.css" integrity="sha384-HSMxcRTRxnN+Bdg0JdbxYKrThecOKuH5zCYotlSAcp1+c8xmyTe9GYg1l9a69psu" crossorigin="anonymous">
|
||||||
|
|
||||||
|
<!-- Optional theme -->
|
||||||
|
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap@3.4.1/dist/css/bootstrap-theme.min.css" integrity="sha384-6pzBo3FDv/PJ8r2KRkGHifhEocL+1X2rVCTTkUfGk7/0pbek5mMa1upzvWbrUbOZ" crossorigin="anonymous">
|
||||||
|
|
||||||
|
<!-- Latest compiled and minified JavaScript -->
|
||||||
|
<script src="https://cdn.jsdelivr.net/npm/bootstrap@3.4.1/dist/js/bootstrap.min.js" integrity="sha384-aJ21OjlMXNL5UyIl/XNwTMqvzeRMZH2w8c5cRVpzpU8Y5bApTppSuUkhZXN0VxHd" crossorigin="anonymous"></script>
|
||||||
|
|
||||||
<title>Q1</title>
|
<title>Q1</title>
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
|
<div class="container">
|
||||||
<h1>Q1: A Simple S3 Browser</h1>
|
<h1>Q1: A Simple S3 Browser</h1>
|
||||||
|
<h2>{{bucket}}/{{path}}</h2>
|
||||||
<h3>Directories</h3>
|
<h3>Directories</h3>
|
||||||
<ul>
|
<ul>
|
||||||
{% for item in dirs %}
|
{% for item in range(dirs|length) %}
|
||||||
<li><a href="/browse/{{bucket}}/{{item}}">{{item}}</a></li>
|
<li><a href="/browse/{{bucket}}/{{dirs[item]}}">{{dirnames[item]}}</a></li>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
</ul>
|
</ul>
|
||||||
<h3>Files</h3>
|
<h3>Files</h3>
|
||||||
<ul>
|
<ul>
|
||||||
{% for item in items %}
|
{% for item in range(items|length) %}
|
||||||
<li>{{item}} -- <a href="/download/{{bucket}}/{{item}}">Download</a></li>
|
<li>{{itemnames[item]}} -- <a href="/download/{{bucket}}/{{items[item]}}">Download</a></li>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
</ul>
|
</ul>
|
||||||
<a href="../"><b>Back</b></a>
|
<a href="../"><b>Back</b></a>
|
||||||
<script type="text/javascript" src="js/materialize.min.js"></script>
|
</div>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
@ -2,24 +2,25 @@
|
|||||||
<html lang="en">
|
<html lang="en">
|
||||||
<head>
|
<head>
|
||||||
<meta charset="UTF-8">
|
<meta charset="UTF-8">
|
||||||
|
<!-- Latest compiled and minified CSS -->
|
||||||
|
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap@3.4.1/dist/css/bootstrap.min.css" integrity="sha384-HSMxcRTRxnN+Bdg0JdbxYKrThecOKuH5zCYotlSAcp1+c8xmyTe9GYg1l9a69psu" crossorigin="anonymous">
|
||||||
|
|
||||||
<!--Import Google Icon Font-->
|
<!-- Optional theme -->
|
||||||
<link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet">
|
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap@3.4.1/dist/css/bootstrap-theme.min.css" integrity="sha384-6pzBo3FDv/PJ8r2KRkGHifhEocL+1X2rVCTTkUfGk7/0pbek5mMa1upzvWbrUbOZ" crossorigin="anonymous">
|
||||||
<!--Import materialize.css-->
|
|
||||||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/materialize/1.0.0/css/materialize.min.css">
|
|
||||||
|
|
||||||
<!--Let browser know website is optimized for mobile-->
|
<!-- Latest compiled and minified JavaScript -->
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
|
<script src="https://cdn.jsdelivr.net/npm/bootstrap@3.4.1/dist/js/bootstrap.min.js" integrity="sha384-aJ21OjlMXNL5UyIl/XNwTMqvzeRMZH2w8c5cRVpzpU8Y5bApTppSuUkhZXN0VxHd" crossorigin="anonymous"></script>
|
||||||
|
|
||||||
<title>Q1</title>
|
<title>Q1</title>
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
|
<div class="container">
|
||||||
<h1>Q1: A Simple S3 Browser</h1>
|
<h1>Q1: A Simple S3 Browser</h1>
|
||||||
<ul>
|
<ul>
|
||||||
{% for bucket in buckets %}
|
{% for bucket in buckets %}
|
||||||
<li><a href="/browse/{{bucket}}">{{bucket}}</a></li>
|
<li><a href="/browse/{{bucket}}">{{bucket}}</a></li>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
</ul>
|
</ul>
|
||||||
<script type="text/javascript" src="js/materialize.min.js"></script>
|
</div>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
Loading…
x
Reference in New Issue
Block a user