use postgresql instead of sqlite, add file size, make shasum optional

Pipfile

@@ -6,6 +6,7 @@ name = "pypi"
 [packages]
 litestar = "*"
 uvicorn = "*"
+psycopg2-binary = "*"
 
 [dev-packages]
 

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "f060f52683f416139dcebc8384332e69264df332180e8a4588d20e89f58fe8fe"
+            "sha256": "83aa26d0270f29f716711058fa169f0e2ccd67751d0c451811440b479bd4f4e3"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -287,6 +287,81 @@
             "markers": "python_version >= '3.8' and python_version < '4.0'",
             "version": "==2.22.2"
         },
+        "psycopg2-binary": {
+            "hashes": [
+                "sha256:04392983d0bb89a8717772a193cfaac58871321e3ec69514e1c4e0d4957b5aff",
+                "sha256:056470c3dc57904bbf63d6f534988bafc4e970ffd50f6271fc4ee7daad9498a5",
+                "sha256:0ea8e3d0ae83564f2fc554955d327fa081d065c8ca5cc6d2abb643e2c9c1200f",
+                "sha256:155e69561d54d02b3c3209545fb08938e27889ff5a10c19de8d23eb5a41be8a5",
+                "sha256:18c5ee682b9c6dd3696dad6e54cc7ff3a1a9020df6a5c0f861ef8bfd338c3ca0",
+                "sha256:19721ac03892001ee8fdd11507e6a2e01f4e37014def96379411ca99d78aeb2c",
+                "sha256:1a6784f0ce3fec4edc64e985865c17778514325074adf5ad8f80636cd029ef7c",
+                "sha256:2286791ececda3a723d1910441c793be44625d86d1a4e79942751197f4d30341",
+                "sha256:230eeae2d71594103cd5b93fd29d1ace6420d0b86f4778739cb1a5a32f607d1f",
+                "sha256:245159e7ab20a71d989da00f280ca57da7641fa2cdcf71749c193cea540a74f7",
+                "sha256:26540d4a9a4e2b096f1ff9cce51253d0504dca5a85872c7f7be23be5a53eb18d",
+                "sha256:270934a475a0e4b6925b5f804e3809dd5f90f8613621d062848dd82f9cd62007",
+                "sha256:27422aa5f11fbcd9b18da48373eb67081243662f9b46e6fd07c3eb46e4535142",
+                "sha256:2ad26b467a405c798aaa1458ba09d7e2b6e5f96b1ce0ac15d82fd9f95dc38a92",
+                "sha256:2b3d2491d4d78b6b14f76881905c7a8a8abcf974aad4a8a0b065273a0ed7a2cb",
+                "sha256:2ce3e21dc3437b1d960521eca599d57408a695a0d3c26797ea0f72e834c7ffe5",
+                "sha256:30e34c4e97964805f715206c7b789d54a78b70f3ff19fbe590104b71c45600e5",
+                "sha256:3216ccf953b3f267691c90c6fe742e45d890d8272326b4a8b20850a03d05b7b8",
+                "sha256:32581b3020c72d7a421009ee1c6bf4a131ef5f0a968fab2e2de0c9d2bb4577f1",
+                "sha256:35958ec9e46432d9076286dda67942ed6d968b9c3a6a2fd62b48939d1d78bf68",
+                "sha256:3abb691ff9e57d4a93355f60d4f4c1dd2d68326c968e7db17ea96df3c023ef73",
+                "sha256:3c18f74eb4386bf35e92ab2354a12c17e5eb4d9798e4c0ad3a00783eae7cd9f1",
+                "sha256:3c4745a90b78e51d9ba06e2088a2fe0c693ae19cc8cb051ccda44e8df8a6eb53",
+                "sha256:3c4ded1a24b20021ebe677b7b08ad10bf09aac197d6943bfe6fec70ac4e4690d",
+                "sha256:3e9c76f0ac6f92ecfc79516a8034a544926430f7b080ec5a0537bca389ee0906",
+                "sha256:48b338f08d93e7be4ab2b5f1dbe69dc5e9ef07170fe1f86514422076d9c010d0",
+                "sha256:4b3df0e6990aa98acda57d983942eff13d824135fe2250e6522edaa782a06de2",
+                "sha256:512d29bb12608891e349af6a0cccedce51677725a921c07dba6342beaf576f9a",
+                "sha256:5a507320c58903967ef7384355a4da7ff3f28132d679aeb23572753cbf2ec10b",
+                "sha256:5c370b1e4975df846b0277b4deba86419ca77dbc25047f535b0bb03d1a544d44",
+                "sha256:6b269105e59ac96aba877c1707c600ae55711d9dcd3fc4b5012e4af68e30c648",
+                "sha256:6d4fa1079cab9018f4d0bd2db307beaa612b0d13ba73b5c6304b9fe2fb441ff7",
+                "sha256:6dc08420625b5a20b53551c50deae6e231e6371194fa0651dbe0fb206452ae1f",
+                "sha256:73aa0e31fa4bb82578f3a6c74a73c273367727de397a7a0f07bd83cbea696baa",
+                "sha256:7559bce4b505762d737172556a4e6ea8a9998ecac1e39b5233465093e8cee697",
+                "sha256:79625966e176dc97ddabc142351e0409e28acf4660b88d1cf6adb876d20c490d",
+                "sha256:7a813c8bdbaaaab1f078014b9b0b13f5de757e2b5d9be6403639b298a04d218b",
+                "sha256:7b2c956c028ea5de47ff3a8d6b3cc3330ab45cf0b7c3da35a2d6ff8420896526",
+                "sha256:7f4152f8f76d2023aac16285576a9ecd2b11a9895373a1f10fd9db54b3ff06b4",
+                "sha256:7f5d859928e635fa3ce3477704acee0f667b3a3d3e4bb109f2b18d4005f38287",
+                "sha256:851485a42dbb0bdc1edcdabdb8557c09c9655dfa2ca0460ff210522e073e319e",
+                "sha256:8608c078134f0b3cbd9f89b34bd60a943b23fd33cc5f065e8d5f840061bd0673",
+                "sha256:880845dfe1f85d9d5f7c412efea7a08946a46894537e4e5d091732eb1d34d9a0",
+                "sha256:8aabf1c1a04584c168984ac678a668094d831f152859d06e055288fa515e4d30",
+                "sha256:8aecc5e80c63f7459a1a2ab2c64df952051df196294d9f739933a9f6687e86b3",
+                "sha256:8cd9b4f2cfab88ed4a9106192de509464b75a906462fb846b936eabe45c2063e",
+                "sha256:8de718c0e1c4b982a54b41779667242bc630b2197948405b7bd8ce16bcecac92",
+                "sha256:9440fa522a79356aaa482aa4ba500b65f28e5d0e63b801abf6aa152a29bd842a",
+                "sha256:b5f86c56eeb91dc3135b3fd8a95dc7ae14c538a2f3ad77a19645cf55bab1799c",
+                "sha256:b73d6d7f0ccdad7bc43e6d34273f70d587ef62f824d7261c4ae9b8b1b6af90e8",
+                "sha256:bb89f0a835bcfc1d42ccd5f41f04870c1b936d8507c6df12b7737febc40f0909",
+                "sha256:c3cc28a6fd5a4a26224007712e79b81dbaee2ffb90ff406256158ec4d7b52b47",
+                "sha256:ce5ab4bf46a211a8e924d307c1b1fcda82368586a19d0a24f8ae166f5c784864",
+                "sha256:d00924255d7fc916ef66e4bf22f354a940c67179ad3fd7067d7a0a9c84d2fbfc",
+                "sha256:d7cd730dfa7c36dbe8724426bf5612798734bff2d3c3857f36f2733f5bfc7c00",
+                "sha256:e217ce4d37667df0bc1c397fdcd8de5e81018ef305aed9415c3b093faaeb10fb",
+                "sha256:e3923c1d9870c49a2d44f795df0c889a22380d36ef92440ff618ec315757e539",
+                "sha256:e5720a5d25e3b99cd0dc5c8a440570469ff82659bb09431c1439b92caf184d3b",
+                "sha256:e8b58f0a96e7a1e341fc894f62c1177a7c83febebb5ff9123b579418fdc8a481",
+                "sha256:e984839e75e0b60cfe75e351db53d6db750b00de45644c5d1f7ee5d1f34a1ce5",
+                "sha256:eb09aa7f9cecb45027683bb55aebaaf45a0df8bf6de68801a6afdc7947bb09d4",
+                "sha256:ec8a77f521a17506a24a5f626cb2aee7850f9b69a0afe704586f63a464f3cd64",
+                "sha256:ecced182e935529727401b24d76634a357c71c9275b356efafd8a2a91ec07392",
+                "sha256:ee0e8c683a7ff25d23b55b11161c2663d4b099770f6085ff0a20d4505778d6b4",
+                "sha256:f0c2d907a1e102526dd2986df638343388b94c33860ff3bbe1384130828714b1",
+                "sha256:f758ed67cab30b9a8d2833609513ce4d3bd027641673d4ebc9c067e4d208eec1",
+                "sha256:f8157bed2f51db683f31306aa497311b560f2265998122abe1dce6428bd86567",
+                "sha256:ffe8ed017e4ed70f68b7b371d84b7d4a790368db9203dfc2d222febd3a9c8863"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.8'",
+            "version": "==2.9.10"
+        },
         "pygments": {
             "hashes": [
                 "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887",

create_table.sql

@@ -1,3 +1,6 @@
+DROP TABLE source CASCADE;
+DROP TABLE shasum CASCADE;
+DROP TABLE fileinfo CASCADE;
 CREATE TABLE source (
     hostname TEXT,
     name TEXT,
@@ -8,7 +11,10 @@ CREATE TABLE shasum (
 );
 CREATE TABLE fileinfo (
     file_path TEXT,
-    source_uuid TEXT,
+    file_size INT NULL,
-    shasum TEXT
+    source_uuid TEXT REFERENCES source (uuid),
+    shasum TEXT REFERENCES shasum (value) NULL,
+    PRIMARY KEY (source_uuid, file_path),
+    date_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP
 );
 

file_index_api/app.py

@@ -1,17 +1,12 @@
 import os
+from pprint import pprint
 from dataclasses import dataclass
 from litestar import Litestar, get, post
-import sqlite3
+from litestar.logging import LoggingConfig
+import psycopg2
+import logging
 
-sql = None
+logger = logging.getLogger()
-if not os.path.isfile('./file_index.db'):
-    with open('create_table.sql') as fh:
-        sql = ' '.join(fh.readlines())
-
-db = sqlite3.connect('./file_index.db')
-
-if sql is not None:
-    db.executescript(sql)
 
 @dataclass
 class PostRequest:
@@ -19,23 +14,95 @@ class PostRequest:
     source: str
     uuid: str
     file_path: str
-    shasum: str
+    file_size: int
+    shasum: str = None
 
 @get("/")
 async def hello_world() -> str:
- return "Hello, world!"
+    return "Hello, world!"
+
+@get("/files/uniq")
+async def files_uniq() -> str:
+    db = psycopg2.connect('dbname=file_index_ian')
+    db.autocommit = True
+    cur = db.cursor()
+    sql = """
+    SELECT * FROM fileinfo
+    """
+    cur.execute(sql)
+    filesnames = {}
+    for elt in cur.fetchall():
+        name = elt[0]
+        name = name[name.rindex('/')+1:]
+        if name not in filesnames:
+            filesnames[name] = 0
+        else:
+            filesnames[name] += 1
+
+
+    duplicates = [
+        (elt, n)
+        for elt, n in filesnames.items()
+        if n > 0
+    ]
+
+    pprint(duplicates)
+    cur.close()
+    db.close()
+
+    return 'ok'
+
 
 @post("/")
-async def register_file(data: PostRequest) -> str:
+def register_file(data: PostRequest) -> str:
- sql = f"""
+    db = psycopg2.connect('dbname=file_index_ian')
- INSERT INTO source VALUES ('{data.hostname}', '{data.source}', '{data.uuid}');
+    db.autocommit = True
- INSERT INTO shasum VALUES ('{data.shasum}');
+    cur = db.cursor()
- INSERT INTO fileinfo VALUES ('{data.file_path}', '{data.uuid}', '{data.shasum}');
+    try:
- """ 
+        sql = f"""
- try:
+        INSERT INTO source VALUES (%s, %s, %s);
-     db.executescript(sql)
+         """ 
- except sqlite3.IntegrityError:
-    pass
- return "ok"
 
-app = Litestar([register_file])
+        cur.execute(sql, (data.hostname, data.source, data.uuid))
+    except psycopg2.errors.UniqueViolation:
+        cur.close()
+        pass
+
+    if data.shasum:
+        try:
+            cur = db.cursor()
+            sql = f"""
+            INSERT INTO shasum VALUES (%s);
+            """
+            cur.execute(sql, (data.shasum, ))
+        except psycopg2.errors.UniqueViolation:
+            cur.close()
+            pass
+    try:
+        cur = db.cursor()
+        if data.shasum:
+            sql = f"""
+            INSERT INTO fileinfo (file_path, file_size, source_uuid, shasum) VALUES (%s, %s, %s, %s);
+            """
+            cur.execute(sql, (data.file_path, data.file_size, data.uuid, data.shasum))
+        else:
+            sql = f"""
+            INSERT INTO fileinfo (file_path, file_size, source_uuid) VALUES (%s, %s, %s);
+            """
+            cur.execute(sql, (data.file_path, data.file_size, data.uuid))
+        cur.close()
+    except psycopg2.errors.UniqueViolation:
+        pass
+    db.close()
+
+
+    logger.info(f"Inserted {data.hostname}:{data.uuid}/{data.file_path} into database.")
+    return f"Inserted {data.hostname}:{data.uuid}/{data.file_path} into database."
+
+app = Litestar([
+    register_file,
+    files_uniq
+], logging_config=LoggingConfig(
+    root={'level': 'DEBUG', 'handlers': ['queue_listener']},
+    log_exceptions='always'
+))

scan_dir.sh

@@ -4,8 +4,18 @@ do
     DEVICE=`findmnt -T "$f" -rn|cut -d' ' -f2`
     UUID=`sudo blkid -o value "$DEVICE"|head -1`
     FILEPATH=`realpath "$f"`
-    SHASUM=`sha256sum "$f"|cut -d' ' -f1`
+
-    JSON="{\"hostname\":\"$HOSTNAME\",\"source\":\"$DEVICE\",\"uuid\":\"$UUID\",\"file_path\":\"$FILEPATH\",\"shasum\":\"$SHASUM\"}"
+    FILESIZE=`du -s0k "$f"|awk '{ print $1}'`
+    JSON="{\"hostname\":\"$HOSTNAME\",\"source\":\"$DEVICE\",\"uuid\":\"$UUID\",\"file_path\":\"$FILEPATH\",\"file_size\":$FILESIZE"
+    if [ $ENABLE_SHASUM ];
+    then
+	    SHASUM=`sha256sum "$f"|cut -d' ' -f1`
+	    JSON="$JSON,\"shasum\":\"$SHASUM\""
+    fi
+    JSON="$JSON}"
+
+    echo $JSON
     curl -X POST --json "$JSON" http://$2/
+    sleep 0.1
 done
 echo