diff --git a/Pipfile b/Pipfile index 3510a81..8e1c3a3 100644 --- a/Pipfile +++ b/Pipfile @@ -6,6 +6,7 @@ name = "pypi" [packages] litestar = "*" uvicorn = "*" +psycopg2-binary = "*" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index 3da6dbd..67de26e 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "f060f52683f416139dcebc8384332e69264df332180e8a4588d20e89f58fe8fe" + "sha256": "83aa26d0270f29f716711058fa169f0e2ccd67751d0c451811440b479bd4f4e3" }, "pipfile-spec": 6, "requires": { @@ -287,6 +287,81 @@ "markers": "python_version >= '3.8' and python_version < '4.0'", "version": "==2.22.2" }, + "psycopg2-binary": { + "hashes": [ + "sha256:04392983d0bb89a8717772a193cfaac58871321e3ec69514e1c4e0d4957b5aff", + "sha256:056470c3dc57904bbf63d6f534988bafc4e970ffd50f6271fc4ee7daad9498a5", + "sha256:0ea8e3d0ae83564f2fc554955d327fa081d065c8ca5cc6d2abb643e2c9c1200f", + "sha256:155e69561d54d02b3c3209545fb08938e27889ff5a10c19de8d23eb5a41be8a5", + "sha256:18c5ee682b9c6dd3696dad6e54cc7ff3a1a9020df6a5c0f861ef8bfd338c3ca0", + "sha256:19721ac03892001ee8fdd11507e6a2e01f4e37014def96379411ca99d78aeb2c", + "sha256:1a6784f0ce3fec4edc64e985865c17778514325074adf5ad8f80636cd029ef7c", + "sha256:2286791ececda3a723d1910441c793be44625d86d1a4e79942751197f4d30341", + "sha256:230eeae2d71594103cd5b93fd29d1ace6420d0b86f4778739cb1a5a32f607d1f", + "sha256:245159e7ab20a71d989da00f280ca57da7641fa2cdcf71749c193cea540a74f7", + "sha256:26540d4a9a4e2b096f1ff9cce51253d0504dca5a85872c7f7be23be5a53eb18d", + "sha256:270934a475a0e4b6925b5f804e3809dd5f90f8613621d062848dd82f9cd62007", + "sha256:27422aa5f11fbcd9b18da48373eb67081243662f9b46e6fd07c3eb46e4535142", + "sha256:2ad26b467a405c798aaa1458ba09d7e2b6e5f96b1ce0ac15d82fd9f95dc38a92", + "sha256:2b3d2491d4d78b6b14f76881905c7a8a8abcf974aad4a8a0b065273a0ed7a2cb", + "sha256:2ce3e21dc3437b1d960521eca599d57408a695a0d3c26797ea0f72e834c7ffe5", + "sha256:30e34c4e97964805f715206c7b789d54a78b70f3ff19fbe590104b71c45600e5", + "sha256:3216ccf953b3f267691c90c6fe742e45d890d8272326b4a8b20850a03d05b7b8", + "sha256:32581b3020c72d7a421009ee1c6bf4a131ef5f0a968fab2e2de0c9d2bb4577f1", + "sha256:35958ec9e46432d9076286dda67942ed6d968b9c3a6a2fd62b48939d1d78bf68", + "sha256:3abb691ff9e57d4a93355f60d4f4c1dd2d68326c968e7db17ea96df3c023ef73", + "sha256:3c18f74eb4386bf35e92ab2354a12c17e5eb4d9798e4c0ad3a00783eae7cd9f1", + "sha256:3c4745a90b78e51d9ba06e2088a2fe0c693ae19cc8cb051ccda44e8df8a6eb53", + "sha256:3c4ded1a24b20021ebe677b7b08ad10bf09aac197d6943bfe6fec70ac4e4690d", + "sha256:3e9c76f0ac6f92ecfc79516a8034a544926430f7b080ec5a0537bca389ee0906", + "sha256:48b338f08d93e7be4ab2b5f1dbe69dc5e9ef07170fe1f86514422076d9c010d0", + "sha256:4b3df0e6990aa98acda57d983942eff13d824135fe2250e6522edaa782a06de2", + "sha256:512d29bb12608891e349af6a0cccedce51677725a921c07dba6342beaf576f9a", + "sha256:5a507320c58903967ef7384355a4da7ff3f28132d679aeb23572753cbf2ec10b", + "sha256:5c370b1e4975df846b0277b4deba86419ca77dbc25047f535b0bb03d1a544d44", + "sha256:6b269105e59ac96aba877c1707c600ae55711d9dcd3fc4b5012e4af68e30c648", + "sha256:6d4fa1079cab9018f4d0bd2db307beaa612b0d13ba73b5c6304b9fe2fb441ff7", + "sha256:6dc08420625b5a20b53551c50deae6e231e6371194fa0651dbe0fb206452ae1f", + "sha256:73aa0e31fa4bb82578f3a6c74a73c273367727de397a7a0f07bd83cbea696baa", + "sha256:7559bce4b505762d737172556a4e6ea8a9998ecac1e39b5233465093e8cee697", + "sha256:79625966e176dc97ddabc142351e0409e28acf4660b88d1cf6adb876d20c490d", + "sha256:7a813c8bdbaaaab1f078014b9b0b13f5de757e2b5d9be6403639b298a04d218b", + "sha256:7b2c956c028ea5de47ff3a8d6b3cc3330ab45cf0b7c3da35a2d6ff8420896526", + "sha256:7f4152f8f76d2023aac16285576a9ecd2b11a9895373a1f10fd9db54b3ff06b4", + "sha256:7f5d859928e635fa3ce3477704acee0f667b3a3d3e4bb109f2b18d4005f38287", + "sha256:851485a42dbb0bdc1edcdabdb8557c09c9655dfa2ca0460ff210522e073e319e", + "sha256:8608c078134f0b3cbd9f89b34bd60a943b23fd33cc5f065e8d5f840061bd0673", + "sha256:880845dfe1f85d9d5f7c412efea7a08946a46894537e4e5d091732eb1d34d9a0", + "sha256:8aabf1c1a04584c168984ac678a668094d831f152859d06e055288fa515e4d30", + "sha256:8aecc5e80c63f7459a1a2ab2c64df952051df196294d9f739933a9f6687e86b3", + "sha256:8cd9b4f2cfab88ed4a9106192de509464b75a906462fb846b936eabe45c2063e", + "sha256:8de718c0e1c4b982a54b41779667242bc630b2197948405b7bd8ce16bcecac92", + "sha256:9440fa522a79356aaa482aa4ba500b65f28e5d0e63b801abf6aa152a29bd842a", + "sha256:b5f86c56eeb91dc3135b3fd8a95dc7ae14c538a2f3ad77a19645cf55bab1799c", + "sha256:b73d6d7f0ccdad7bc43e6d34273f70d587ef62f824d7261c4ae9b8b1b6af90e8", + "sha256:bb89f0a835bcfc1d42ccd5f41f04870c1b936d8507c6df12b7737febc40f0909", + "sha256:c3cc28a6fd5a4a26224007712e79b81dbaee2ffb90ff406256158ec4d7b52b47", + "sha256:ce5ab4bf46a211a8e924d307c1b1fcda82368586a19d0a24f8ae166f5c784864", + "sha256:d00924255d7fc916ef66e4bf22f354a940c67179ad3fd7067d7a0a9c84d2fbfc", + "sha256:d7cd730dfa7c36dbe8724426bf5612798734bff2d3c3857f36f2733f5bfc7c00", + "sha256:e217ce4d37667df0bc1c397fdcd8de5e81018ef305aed9415c3b093faaeb10fb", + "sha256:e3923c1d9870c49a2d44f795df0c889a22380d36ef92440ff618ec315757e539", + "sha256:e5720a5d25e3b99cd0dc5c8a440570469ff82659bb09431c1439b92caf184d3b", + "sha256:e8b58f0a96e7a1e341fc894f62c1177a7c83febebb5ff9123b579418fdc8a481", + "sha256:e984839e75e0b60cfe75e351db53d6db750b00de45644c5d1f7ee5d1f34a1ce5", + "sha256:eb09aa7f9cecb45027683bb55aebaaf45a0df8bf6de68801a6afdc7947bb09d4", + "sha256:ec8a77f521a17506a24a5f626cb2aee7850f9b69a0afe704586f63a464f3cd64", + "sha256:ecced182e935529727401b24d76634a357c71c9275b356efafd8a2a91ec07392", + "sha256:ee0e8c683a7ff25d23b55b11161c2663d4b099770f6085ff0a20d4505778d6b4", + "sha256:f0c2d907a1e102526dd2986df638343388b94c33860ff3bbe1384130828714b1", + "sha256:f758ed67cab30b9a8d2833609513ce4d3bd027641673d4ebc9c067e4d208eec1", + "sha256:f8157bed2f51db683f31306aa497311b560f2265998122abe1dce6428bd86567", + "sha256:ffe8ed017e4ed70f68b7b371d84b7d4a790368db9203dfc2d222febd3a9c8863" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==2.9.10" + }, "pygments": { "hashes": [ "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", diff --git a/create_table.sql b/create_table.sql index f725bc8..48ff098 100644 --- a/create_table.sql +++ b/create_table.sql @@ -1,3 +1,6 @@ +DROP TABLE source CASCADE; +DROP TABLE shasum CASCADE; +DROP TABLE fileinfo CASCADE; CREATE TABLE source ( hostname TEXT, name TEXT, @@ -8,7 +11,10 @@ CREATE TABLE shasum ( ); CREATE TABLE fileinfo ( file_path TEXT, - source_uuid TEXT, - shasum TEXT + file_size INT NULL, + source_uuid TEXT REFERENCES source (uuid), + shasum TEXT REFERENCES shasum (value) NULL, + PRIMARY KEY (source_uuid, file_path), + date_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP ); diff --git a/file_index_api/app.py b/file_index_api/app.py index b9cab60..1e53af9 100644 --- a/file_index_api/app.py +++ b/file_index_api/app.py @@ -1,17 +1,12 @@ import os +from pprint import pprint from dataclasses import dataclass from litestar import Litestar, get, post -import sqlite3 +from litestar.logging import LoggingConfig +import psycopg2 +import logging -sql = None -if not os.path.isfile('./file_index.db'): - with open('create_table.sql') as fh: - sql = ' '.join(fh.readlines()) - -db = sqlite3.connect('./file_index.db') - -if sql is not None: - db.executescript(sql) +logger = logging.getLogger() @dataclass class PostRequest: @@ -19,23 +14,95 @@ class PostRequest: source: str uuid: str file_path: str - shasum: str + file_size: int + shasum: str = None @get("/") async def hello_world() -> str: - return "Hello, world!" + return "Hello, world!" + +@get("/files/uniq") +async def files_uniq() -> str: + db = psycopg2.connect('dbname=file_index_ian') + db.autocommit = True + cur = db.cursor() + sql = """ + SELECT * FROM fileinfo + """ + cur.execute(sql) + filesnames = {} + for elt in cur.fetchall(): + name = elt[0] + name = name[name.rindex('/')+1:] + if name not in filesnames: + filesnames[name] = 0 + else: + filesnames[name] += 1 + + + duplicates = [ + (elt, n) + for elt, n in filesnames.items() + if n > 0 + ] + + pprint(duplicates) + cur.close() + db.close() + + return 'ok' + @post("/") -async def register_file(data: PostRequest) -> str: - sql = f""" - INSERT INTO source VALUES ('{data.hostname}', '{data.source}', '{data.uuid}'); - INSERT INTO shasum VALUES ('{data.shasum}'); - INSERT INTO fileinfo VALUES ('{data.file_path}', '{data.uuid}', '{data.shasum}'); - """ - try: - db.executescript(sql) - except sqlite3.IntegrityError: - pass - return "ok" +def register_file(data: PostRequest) -> str: + db = psycopg2.connect('dbname=file_index_ian') + db.autocommit = True + cur = db.cursor() + try: + sql = f""" + INSERT INTO source VALUES (%s, %s, %s); + """ -app = Litestar([register_file]) + cur.execute(sql, (data.hostname, data.source, data.uuid)) + except psycopg2.errors.UniqueViolation: + cur.close() + pass + + if data.shasum: + try: + cur = db.cursor() + sql = f""" + INSERT INTO shasum VALUES (%s); + """ + cur.execute(sql, (data.shasum, )) + except psycopg2.errors.UniqueViolation: + cur.close() + pass + try: + cur = db.cursor() + if data.shasum: + sql = f""" + INSERT INTO fileinfo (file_path, file_size, source_uuid, shasum) VALUES (%s, %s, %s, %s); + """ + cur.execute(sql, (data.file_path, data.file_size, data.uuid, data.shasum)) + else: + sql = f""" + INSERT INTO fileinfo (file_path, file_size, source_uuid) VALUES (%s, %s, %s); + """ + cur.execute(sql, (data.file_path, data.file_size, data.uuid)) + cur.close() + except psycopg2.errors.UniqueViolation: + pass + db.close() + + + logger.info(f"Inserted {data.hostname}:{data.uuid}/{data.file_path} into database.") + return f"Inserted {data.hostname}:{data.uuid}/{data.file_path} into database." + +app = Litestar([ + register_file, + files_uniq +], logging_config=LoggingConfig( + root={'level': 'DEBUG', 'handlers': ['queue_listener']}, + log_exceptions='always' +)) diff --git a/scan_dir.sh b/scan_dir.sh index 029b0cf..2073fda 100755 --- a/scan_dir.sh +++ b/scan_dir.sh @@ -4,8 +4,18 @@ do DEVICE=`findmnt -T "$f" -rn|cut -d' ' -f2` UUID=`sudo blkid -o value "$DEVICE"|head -1` FILEPATH=`realpath "$f"` - SHASUM=`sha256sum "$f"|cut -d' ' -f1` - JSON="{\"hostname\":\"$HOSTNAME\",\"source\":\"$DEVICE\",\"uuid\":\"$UUID\",\"file_path\":\"$FILEPATH\",\"shasum\":\"$SHASUM\"}" + + FILESIZE=`du -s0k "$f"|awk '{ print $1}'` + JSON="{\"hostname\":\"$HOSTNAME\",\"source\":\"$DEVICE\",\"uuid\":\"$UUID\",\"file_path\":\"$FILEPATH\",\"file_size\":$FILESIZE" + if [ $ENABLE_SHASUM ]; + then + SHASUM=`sha256sum "$f"|cut -d' ' -f1` + JSON="$JSON,\"shasum\":\"$SHASUM\"" + fi + JSON="$JSON}" + + echo $JSON curl -X POST --json "$JSON" http://$2/ + sleep 0.1 done echo