use postgresql instead of sqlite, add file size, make shasum optional
This commit is contained in:
emixam
parent
3c820ac3de
commit
78fa3445e7
1
Pipfile
1
Pipfile
@ -6,6 +6,7 @@ name = "pypi"
|
||||
[packages]
|
||||
litestar = "*"
|
||||
uvicorn = "*"
|
||||
psycopg2-binary = "*"
|
||||
|
||||
[dev-packages]
|
||||
|
||||
|
||||
77
Pipfile.lock
generated
77
Pipfile.lock
generated
@ -1,7 +1,7 @@
|
||||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "f060f52683f416139dcebc8384332e69264df332180e8a4588d20e89f58fe8fe"
|
||||
"sha256": "83aa26d0270f29f716711058fa169f0e2ccd67751d0c451811440b479bd4f4e3"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
@ -287,6 +287,81 @@
|
||||
"markers": "python_version >= '3.8' and python_version < '4.0'",
|
||||
"version": "==2.22.2"
|
||||
},
|
||||
"psycopg2-binary": {
|
||||
"hashes": [
|
||||
"sha256:04392983d0bb89a8717772a193cfaac58871321e3ec69514e1c4e0d4957b5aff",
|
||||
"sha256:056470c3dc57904bbf63d6f534988bafc4e970ffd50f6271fc4ee7daad9498a5",
|
||||
"sha256:0ea8e3d0ae83564f2fc554955d327fa081d065c8ca5cc6d2abb643e2c9c1200f",
|
||||
"sha256:155e69561d54d02b3c3209545fb08938e27889ff5a10c19de8d23eb5a41be8a5",
|
||||
"sha256:18c5ee682b9c6dd3696dad6e54cc7ff3a1a9020df6a5c0f861ef8bfd338c3ca0",
|
||||
"sha256:19721ac03892001ee8fdd11507e6a2e01f4e37014def96379411ca99d78aeb2c",
|
||||
"sha256:1a6784f0ce3fec4edc64e985865c17778514325074adf5ad8f80636cd029ef7c",
|
||||
"sha256:2286791ececda3a723d1910441c793be44625d86d1a4e79942751197f4d30341",
|
||||
"sha256:230eeae2d71594103cd5b93fd29d1ace6420d0b86f4778739cb1a5a32f607d1f",
|
||||
"sha256:245159e7ab20a71d989da00f280ca57da7641fa2cdcf71749c193cea540a74f7",
|
||||
"sha256:26540d4a9a4e2b096f1ff9cce51253d0504dca5a85872c7f7be23be5a53eb18d",
|
||||
"sha256:270934a475a0e4b6925b5f804e3809dd5f90f8613621d062848dd82f9cd62007",
|
||||
"sha256:27422aa5f11fbcd9b18da48373eb67081243662f9b46e6fd07c3eb46e4535142",
|
||||
"sha256:2ad26b467a405c798aaa1458ba09d7e2b6e5f96b1ce0ac15d82fd9f95dc38a92",
|
||||
"sha256:2b3d2491d4d78b6b14f76881905c7a8a8abcf974aad4a8a0b065273a0ed7a2cb",
|
||||
"sha256:2ce3e21dc3437b1d960521eca599d57408a695a0d3c26797ea0f72e834c7ffe5",
|
||||
"sha256:30e34c4e97964805f715206c7b789d54a78b70f3ff19fbe590104b71c45600e5",
|
||||
"sha256:3216ccf953b3f267691c90c6fe742e45d890d8272326b4a8b20850a03d05b7b8",
|
||||
"sha256:32581b3020c72d7a421009ee1c6bf4a131ef5f0a968fab2e2de0c9d2bb4577f1",
|
||||
"sha256:35958ec9e46432d9076286dda67942ed6d968b9c3a6a2fd62b48939d1d78bf68",
|
||||
"sha256:3abb691ff9e57d4a93355f60d4f4c1dd2d68326c968e7db17ea96df3c023ef73",
|
||||
"sha256:3c18f74eb4386bf35e92ab2354a12c17e5eb4d9798e4c0ad3a00783eae7cd9f1",
|
||||
"sha256:3c4745a90b78e51d9ba06e2088a2fe0c693ae19cc8cb051ccda44e8df8a6eb53",
|
||||
"sha256:3c4ded1a24b20021ebe677b7b08ad10bf09aac197d6943bfe6fec70ac4e4690d",
|
||||
"sha256:3e9c76f0ac6f92ecfc79516a8034a544926430f7b080ec5a0537bca389ee0906",
|
||||
"sha256:48b338f08d93e7be4ab2b5f1dbe69dc5e9ef07170fe1f86514422076d9c010d0",
|
||||
"sha256:4b3df0e6990aa98acda57d983942eff13d824135fe2250e6522edaa782a06de2",
|
||||
"sha256:512d29bb12608891e349af6a0cccedce51677725a921c07dba6342beaf576f9a",
|
||||
"sha256:5a507320c58903967ef7384355a4da7ff3f28132d679aeb23572753cbf2ec10b",
|
||||
"sha256:5c370b1e4975df846b0277b4deba86419ca77dbc25047f535b0bb03d1a544d44",
|
||||
"sha256:6b269105e59ac96aba877c1707c600ae55711d9dcd3fc4b5012e4af68e30c648",
|
||||
"sha256:6d4fa1079cab9018f4d0bd2db307beaa612b0d13ba73b5c6304b9fe2fb441ff7",
|
||||
"sha256:6dc08420625b5a20b53551c50deae6e231e6371194fa0651dbe0fb206452ae1f",
|
||||
"sha256:73aa0e31fa4bb82578f3a6c74a73c273367727de397a7a0f07bd83cbea696baa",
|
||||
"sha256:7559bce4b505762d737172556a4e6ea8a9998ecac1e39b5233465093e8cee697",
|
||||
"sha256:79625966e176dc97ddabc142351e0409e28acf4660b88d1cf6adb876d20c490d",
|
||||
"sha256:7a813c8bdbaaaab1f078014b9b0b13f5de757e2b5d9be6403639b298a04d218b",
|
||||
"sha256:7b2c956c028ea5de47ff3a8d6b3cc3330ab45cf0b7c3da35a2d6ff8420896526",
|
||||
"sha256:7f4152f8f76d2023aac16285576a9ecd2b11a9895373a1f10fd9db54b3ff06b4",
|
||||
"sha256:7f5d859928e635fa3ce3477704acee0f667b3a3d3e4bb109f2b18d4005f38287",
|
||||
"sha256:851485a42dbb0bdc1edcdabdb8557c09c9655dfa2ca0460ff210522e073e319e",
|
||||
"sha256:8608c078134f0b3cbd9f89b34bd60a943b23fd33cc5f065e8d5f840061bd0673",
|
||||
"sha256:880845dfe1f85d9d5f7c412efea7a08946a46894537e4e5d091732eb1d34d9a0",
|
||||
"sha256:8aabf1c1a04584c168984ac678a668094d831f152859d06e055288fa515e4d30",
|
||||
"sha256:8aecc5e80c63f7459a1a2ab2c64df952051df196294d9f739933a9f6687e86b3",
|
||||
"sha256:8cd9b4f2cfab88ed4a9106192de509464b75a906462fb846b936eabe45c2063e",
|
||||
"sha256:8de718c0e1c4b982a54b41779667242bc630b2197948405b7bd8ce16bcecac92",
|
||||
"sha256:9440fa522a79356aaa482aa4ba500b65f28e5d0e63b801abf6aa152a29bd842a",
|
||||
"sha256:b5f86c56eeb91dc3135b3fd8a95dc7ae14c538a2f3ad77a19645cf55bab1799c",
|
||||
"sha256:b73d6d7f0ccdad7bc43e6d34273f70d587ef62f824d7261c4ae9b8b1b6af90e8",
|
||||
"sha256:bb89f0a835bcfc1d42ccd5f41f04870c1b936d8507c6df12b7737febc40f0909",
|
||||
"sha256:c3cc28a6fd5a4a26224007712e79b81dbaee2ffb90ff406256158ec4d7b52b47",
|
||||
"sha256:ce5ab4bf46a211a8e924d307c1b1fcda82368586a19d0a24f8ae166f5c784864",
|
||||
"sha256:d00924255d7fc916ef66e4bf22f354a940c67179ad3fd7067d7a0a9c84d2fbfc",
|
||||
"sha256:d7cd730dfa7c36dbe8724426bf5612798734bff2d3c3857f36f2733f5bfc7c00",
|
||||
"sha256:e217ce4d37667df0bc1c397fdcd8de5e81018ef305aed9415c3b093faaeb10fb",
|
||||
"sha256:e3923c1d9870c49a2d44f795df0c889a22380d36ef92440ff618ec315757e539",
|
||||
"sha256:e5720a5d25e3b99cd0dc5c8a440570469ff82659bb09431c1439b92caf184d3b",
|
||||
"sha256:e8b58f0a96e7a1e341fc894f62c1177a7c83febebb5ff9123b579418fdc8a481",
|
||||
"sha256:e984839e75e0b60cfe75e351db53d6db750b00de45644c5d1f7ee5d1f34a1ce5",
|
||||
"sha256:eb09aa7f9cecb45027683bb55aebaaf45a0df8bf6de68801a6afdc7947bb09d4",
|
||||
"sha256:ec8a77f521a17506a24a5f626cb2aee7850f9b69a0afe704586f63a464f3cd64",
|
||||
"sha256:ecced182e935529727401b24d76634a357c71c9275b356efafd8a2a91ec07392",
|
||||
"sha256:ee0e8c683a7ff25d23b55b11161c2663d4b099770f6085ff0a20d4505778d6b4",
|
||||
"sha256:f0c2d907a1e102526dd2986df638343388b94c33860ff3bbe1384130828714b1",
|
||||
"sha256:f758ed67cab30b9a8d2833609513ce4d3bd027641673d4ebc9c067e4d208eec1",
|
||||
"sha256:f8157bed2f51db683f31306aa497311b560f2265998122abe1dce6428bd86567",
|
||||
"sha256:ffe8ed017e4ed70f68b7b371d84b7d4a790368db9203dfc2d222febd3a9c8863"
|
||||
],
|
||||
"index": "pypi",
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==2.9.10"
|
||||
},
|
||||
"pygments": {
|
||||
"hashes": [
|
||||
"sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887",
|
||||
|
||||
@ -1,3 +1,6 @@
|
||||
DROP TABLE source CASCADE;
|
||||
DROP TABLE shasum CASCADE;
|
||||
DROP TABLE fileinfo CASCADE;
|
||||
CREATE TABLE source (
|
||||
hostname TEXT,
|
||||
name TEXT,
|
||||
@ -8,7 +11,10 @@ CREATE TABLE shasum (
|
||||
);
|
||||
CREATE TABLE fileinfo (
|
||||
file_path TEXT,
|
||||
source_uuid TEXT,
|
||||
shasum TEXT
|
||||
file_size INT NULL,
|
||||
source_uuid TEXT REFERENCES source (uuid),
|
||||
shasum TEXT REFERENCES shasum (value) NULL,
|
||||
PRIMARY KEY (source_uuid, file_path),
|
||||
date_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
|
||||
@ -1,17 +1,12 @@
|
||||
import os
|
||||
from pprint import pprint
|
||||
from dataclasses import dataclass
|
||||
from litestar import Litestar, get, post
|
||||
import sqlite3
|
||||
from litestar.logging import LoggingConfig
|
||||
import psycopg2
|
||||
import logging
|
||||
|
||||
sql = None
|
||||
if not os.path.isfile('./file_index.db'):
|
||||
with open('create_table.sql') as fh:
|
||||
sql = ' '.join(fh.readlines())
|
||||
|
||||
db = sqlite3.connect('./file_index.db')
|
||||
|
||||
if sql is not None:
|
||||
db.executescript(sql)
|
||||
logger = logging.getLogger()
|
||||
|
||||
@dataclass
|
||||
class PostRequest:
|
||||
@ -19,23 +14,95 @@ class PostRequest:
|
||||
source: str
|
||||
uuid: str
|
||||
file_path: str
|
||||
shasum: str
|
||||
file_size: int
|
||||
shasum: str = None
|
||||
|
||||
@get("/")
|
||||
async def hello_world() -> str:
|
||||
return "Hello, world!"
|
||||
return "Hello, world!"
|
||||
|
||||
@get("/files/uniq")
|
||||
async def files_uniq() -> str:
|
||||
db = psycopg2.connect('dbname=file_index_ian')
|
||||
db.autocommit = True
|
||||
cur = db.cursor()
|
||||
sql = """
|
||||
SELECT * FROM fileinfo
|
||||
"""
|
||||
cur.execute(sql)
|
||||
filesnames = {}
|
||||
for elt in cur.fetchall():
|
||||
name = elt[0]
|
||||
name = name[name.rindex('/')+1:]
|
||||
if name not in filesnames:
|
||||
filesnames[name] = 0
|
||||
else:
|
||||
filesnames[name] += 1
|
||||
|
||||
|
||||
duplicates = [
|
||||
(elt, n)
|
||||
for elt, n in filesnames.items()
|
||||
if n > 0
|
||||
]
|
||||
|
||||
pprint(duplicates)
|
||||
cur.close()
|
||||
db.close()
|
||||
|
||||
return 'ok'
|
||||
|
||||
|
||||
@post("/")
|
||||
async def register_file(data: PostRequest) -> str:
|
||||
sql = f"""
|
||||
INSERT INTO source VALUES ('{data.hostname}', '{data.source}', '{data.uuid}');
|
||||
INSERT INTO shasum VALUES ('{data.shasum}');
|
||||
INSERT INTO fileinfo VALUES ('{data.file_path}', '{data.uuid}', '{data.shasum}');
|
||||
"""
|
||||
try:
|
||||
db.executescript(sql)
|
||||
except sqlite3.IntegrityError:
|
||||
pass
|
||||
return "ok"
|
||||
def register_file(data: PostRequest) -> str:
|
||||
db = psycopg2.connect('dbname=file_index_ian')
|
||||
db.autocommit = True
|
||||
cur = db.cursor()
|
||||
try:
|
||||
sql = f"""
|
||||
INSERT INTO source VALUES (%s, %s, %s);
|
||||
"""
|
||||
|
||||
app = Litestar([register_file])
|
||||
cur.execute(sql, (data.hostname, data.source, data.uuid))
|
||||
except psycopg2.errors.UniqueViolation:
|
||||
cur.close()
|
||||
pass
|
||||
|
||||
if data.shasum:
|
||||
try:
|
||||
cur = db.cursor()
|
||||
sql = f"""
|
||||
INSERT INTO shasum VALUES (%s);
|
||||
"""
|
||||
cur.execute(sql, (data.shasum, ))
|
||||
except psycopg2.errors.UniqueViolation:
|
||||
cur.close()
|
||||
pass
|
||||
try:
|
||||
cur = db.cursor()
|
||||
if data.shasum:
|
||||
sql = f"""
|
||||
INSERT INTO fileinfo (file_path, file_size, source_uuid, shasum) VALUES (%s, %s, %s, %s);
|
||||
"""
|
||||
cur.execute(sql, (data.file_path, data.file_size, data.uuid, data.shasum))
|
||||
else:
|
||||
sql = f"""
|
||||
INSERT INTO fileinfo (file_path, file_size, source_uuid) VALUES (%s, %s, %s);
|
||||
"""
|
||||
cur.execute(sql, (data.file_path, data.file_size, data.uuid))
|
||||
cur.close()
|
||||
except psycopg2.errors.UniqueViolation:
|
||||
pass
|
||||
db.close()
|
||||
|
||||
|
||||
logger.info(f"Inserted {data.hostname}:{data.uuid}/{data.file_path} into database.")
|
||||
return f"Inserted {data.hostname}:{data.uuid}/{data.file_path} into database."
|
||||
|
||||
app = Litestar([
|
||||
register_file,
|
||||
files_uniq
|
||||
], logging_config=LoggingConfig(
|
||||
root={'level': 'DEBUG', 'handlers': ['queue_listener']},
|
||||
log_exceptions='always'
|
||||
))
|
||||
|
||||
14
scan_dir.sh
14
scan_dir.sh
@ -4,8 +4,18 @@ do
|
||||
DEVICE=`findmnt -T "$f" -rn|cut -d' ' -f2`
|
||||
UUID=`sudo blkid -o value "$DEVICE"|head -1`
|
||||
FILEPATH=`realpath "$f"`
|
||||
SHASUM=`sha256sum "$f"|cut -d' ' -f1`
|
||||
JSON="{\"hostname\":\"$HOSTNAME\",\"source\":\"$DEVICE\",\"uuid\":\"$UUID\",\"file_path\":\"$FILEPATH\",\"shasum\":\"$SHASUM\"}"
|
||||
|
||||
FILESIZE=`du -s0k "$f"|awk '{ print $1}'`
|
||||
JSON="{\"hostname\":\"$HOSTNAME\",\"source\":\"$DEVICE\",\"uuid\":\"$UUID\",\"file_path\":\"$FILEPATH\",\"file_size\":$FILESIZE"
|
||||
if [ $ENABLE_SHASUM ];
|
||||
then
|
||||
SHASUM=`sha256sum "$f"|cut -d' ' -f1`
|
||||
JSON="$JSON,\"shasum\":\"$SHASUM\""
|
||||
fi
|
||||
JSON="$JSON}"
|
||||
|
||||
echo $JSON
|
||||
curl -X POST --json "$JSON" http://$2/
|
||||
sleep 0.1
|
||||
done
|
||||
echo
|
||||
|
||||
Loading…
Reference in New Issue
Block a user