Add support for expiring files

SUPPLEMENTALLY:
- Add an `expiration` field to the `file` table of the database
- Produce a migration for the above change
- Overhaul the cleanup script, and integrate into fhost.py
  (now run using FLASK_APP=fhost flask prune)
- Replace the old cleanup script with a deprecation notice
- Add information about how to expire files to the index
- Update the README with information about the new script

Squashed commits:

Add a note explaining that expired files aren't immediately removed

Show correct times on the index page graph

Improve the migration script, removing the need for --legacy

Use automap in place of an explicit file map in migration

Remove vestigial `touch()`

Don't crash when upgrading a fresh database

Remove vestigial warning about legacy files

More efficiently filter to unexpired files when migrating

https://git.0x0.st/mia/0x0/pulls/72#issuecomment-224

Coalesce updates to the database during migration

https://git.0x0.st/mia/0x0/pulls/72#issuecomment-226

Remove vestigial database model

https://git.0x0.st/mia/0x0/pulls/72#issuecomment-261

prune:  Stream expired files from the database

(as opposed to collecting them all first)

config.example.py:  Add min & max expiration + description

README.rst

@@ -35,8 +35,8 @@ downsides, one of them being that range requests will not work. This is a
 problem for example when streaming media files: It won’t be possible to seek,
 and some ISOBMFF (MP4) files will not play at all.
 
-To make files expire, simply create a cronjob that runs ``cleanup.py`` every
+To make files expire, simply create a cronjob that runs ``FLASK_APP=fhost
-now and then.
+flask prune`` every now and then.
 
 Before running the service for the first time, run ``FLASK_APP=fhost flask db upgrade``.
 

cleanup.py

@@ -1,44 +1,8 @@
 #!/usr/bin/env python3
 
-"""
+print("This script has been replaced!!")
-    Copyright © 2020 Mia Herkt
+print("Instead, please run")
-    Licensed under the EUPL, Version 1.2 or - as soon as approved
+print("")
-    by the European Commission - subsequent versions of the EUPL
+print("   $ FLASK_APP=fhost flask prune")
-    (the "License");
+print("")
-    You may not use this work except in compliance with the License.
+exit(1);
-    You may obtain a copy of the license at:
-
-        https://joinup.ec.europa.eu/software/page/eupl
-
-    Unless required by applicable law or agreed to in writing,
-    software distributed under the License is distributed on an
-    "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
-    either express or implied.
-    See the License for the specific language governing permissions
-    and limitations under the License.
-"""
-
-import os
-import sys
-import time
-import datetime
-from fhost import app
-
-os.chdir(os.path.dirname(sys.argv[0]))
-os.chdir(app.config["FHOST_STORAGE_PATH"])
-
-files = [f for f in os.listdir(".")]
-
-maxs = app.config["MAX_CONTENT_LENGTH"]
-mind = 30
-maxd = 365
-
-for f in files:
-    stat = os.stat(f)
-    systime = time.time()
-    age = datetime.timedelta(seconds=(systime - stat.st_mtime)).days
-
-    maxage = mind + (-maxd + mind) * (stat.st_size / maxs - 1) ** 3
-
-    if age >= maxage:
-        os.remove(f)

fhost.py

@@ -22,12 +22,17 @@
 from flask import Flask, abort, make_response, redirect, request, send_from_directory, url_for, Response, render_template
 from flask_sqlalchemy import SQLAlchemy
 from flask_migrate import Migrate
+from sqlalchemy import and_
 from jinja2.exceptions import *
 from jinja2 import ChoiceLoader, FileSystemLoader
 from hashlib import sha256
 from magic import Magic
 from mimetypes import guess_extension
+import click
+import os
 import sys
+import time
+import typing
 import requests
 from validators import url as url_valid
 from pathlib import Path
@@ -121,12 +126,14 @@ class File(db.Model):
     addr = db.Column(db.UnicodeText)
     removed = db.Column(db.Boolean, default=False)
     nsfw_score = db.Column(db.Float)
+    expiration = db.Column(db.BigInteger)
 
-    def __init__(self, sha256, ext, mime, addr):
+    def __init__(self, sha256, ext, mime, addr, expiration):
         self.sha256 = sha256
         self.ext = ext
         self.mime = mime
         self.addr = addr
+        self.expiration = expiration
 
     def getname(self):
         return u"{0}{1}".format(su.enbase(self.id), self.ext)
@@ -139,7 +146,16 @@ class File(db.Model):
         else:
             return url_for("get", path=n, _external=True) + "\n"
 
-    def store(file_, addr):
+    """
+    requested_expiration can be:
+        - None, to use the longest allowed file lifespan
+        - a duration (in hours) that the file should live for
+        - a timestamp in epoch millis that the file should expire at
+
+    Any value greater that the longest allowed file lifespan will be rounded down to that
+    value.
+    """
+    def store(file_, requested_expiration: typing.Optional[int], addr):
         data = file_.read()
         digest = sha256(data).hexdigest()
 
@@ -175,15 +191,51 @@ class File(db.Model):
 
             return ext[:app.config["FHOST_MAX_EXT_LENGTH"]] or ".bin"
 
-        f = File.query.filter_by(sha256=digest).first()
+        # Returns the epoch millisecond that this file should expire
+        #
+        # Uses the expiration time provided by the user (requested_expiration)
+        # upper-bounded by an algorithm that computes the size based on the size of the
+        # file.
+        #
+        # That is, all files are assigned a computed expiration, which can voluntarily
+        # shortened by the user either by providing a timestamp in epoch millis or a
+        # duration in hours.
+        def get_expiration() -> int:
+            current_epoch_millis = time.time() * 1000;
 
+            # Maximum lifetime of the file in milliseconds
+            this_files_max_lifespan = get_max_lifespan(len(data));
+
+            # The latest allowed expiration date for this file, in epoch millis
+            this_files_max_expiration = this_files_max_lifespan + 1000 * time.time();
+
+            if requested_expiration is None:
+                return this_files_max_expiration
+            elif requested_expiration < 1650460320000:
+                # Treat the requested expiration time as a duration in hours
+                requested_expiration_ms = requested_expiration * 60 * 60 * 1000
+                return min(this_files_max_expiration, current_epoch_millis + requested_expiration_ms)
+            else:
+                # Treat the requested expiration time as a timestamp in epoch millis
+                return min(this_files_max_expiration, requested_expiration);
+
+        f = File.query.filter_by(sha256=digest).first()
         if f:
+            # If the file already exists
             if f.removed:
+                # The file was removed by moderation, so don't accept it back
                 abort(451)
+            if f.expiration is None:
+                # The file has expired, so give it a new expiration date
+                f.expiration = get_expiration()
+            else:
+                # The file already exists, update the expiration if needed
+                f.expiration = max(f.expiration, get_expiration())
         else:
             mime = get_mime()
             ext = get_ext(mime)
-            f = File(digest, ext, mime, addr)
+            expiration = get_expiration()
+            f = File(digest, ext, mime, addr, expiration)
 
         f.addr = addr
 
@@ -194,8 +246,6 @@ class File(db.Model):
         if not p.is_file():
             with open(p, "wb") as of:
                 of.write(data)
-        else:
-            p.touch()
 
         if not f.nsfw_score and app.config["NSFW_DETECT"]:
             f.nsfw_score = nsfw.detect(p)
@@ -260,11 +310,20 @@ def in_upload_bl(addr):
 
     return False
 
-def store_file(f, addr):
+"""
+requested_expiration can be:
+    - None, to use the longest allowed file lifespan
+    - a duration (in hours) that the file should live for
+    - a timestamp in epoch millis that the file should expire at
+
+Any value greater that the longest allowed file lifespan will be rounded down to that
+value.
+"""
+def store_file(f, requested_expiration:  typing.Optional[int], addr):
     if in_upload_bl(addr):
         return "Your host is blocked from uploading files.\n", 451
 
-    sf = File.store(f, addr)
+    sf = File.store(f, requested_expiration, addr)
 
     return sf.geturl()
 
@@ -289,7 +348,7 @@ def store_url(url, addr):
 
             f = urlfile(read=r.raw.read, content_type=r.headers["content-type"], filename="")
 
-            return store_file(f, addr)
+            return store_file(f, None, addr)
         else:
             abort(413)
     else:
@@ -336,7 +395,23 @@ def fhost():
         sf = None
 
         if "file" in request.files:
-            return store_file(request.files["file"], request.remote_addr)
+            try:
+                # Store the file with the requested expiration date
+                return store_file(
+                    request.files["file"],
+                    int(request.form["expires"]),
+                    request.remote_addr
+                )
+            except ValueError:
+                # The requested expiration date wasn't properly formed
+                abort(400)
+            except KeyError:
+                # No expiration date was requested, store with the max lifespan
+                return store_file(
+                    request.files["file"],
+                    None,
+                    request.remote_addr
+                )
         elif "url" in request.form:
             return store_url(request.form["url"], request.remote_addr)
         elif "shorten" in request.form:
@@ -364,3 +439,73 @@ def ehandler(e):
         return render_template(f"{e.code}.html", id=id), e.code
     except TemplateNotFound:
         return "Segmentation fault\n", e.code
+
+@app.cli.command("prune")
+def prune():
+    """
+    Clean up expired files
+
+    Deletes any files from the filesystem which have hit their expiration time.  This
+    doesn't remove them from the database, only from the filesystem.  It's recommended
+    that server owners run this command regularly, or set it up on a timer.
+    """
+    current_time = time.time() * 1000;
+
+    # The path to where uploaded files are stored
+    storage = Path(app.config["FHOST_STORAGE_PATH"])
+
+    # A list of all files who've passed their expiration times
+    expired_files = File.query\
+        .where(
+            and_(
+                File.expiration.is_not(None),
+                File.expiration < current_time
+            )
+        )
+
+    files_removed = 0;
+
+    # For every expired file...
+    for file in expired_files:
+        # Log the file we're about to remove
+        file_name = file.getname()
+        file_hash = file.sha256
+        file_path = storage / file_hash
+        print(f"Removing expired file {file_name} [{file_hash}]")
+
+        # Remove it from the file system
+        try:
+            os.remove(file_path)
+            files_removed += 1;
+        except FileNotFoundError:
+            pass # If the file was already gone, we're good
+        except OSError as e:
+            print(e)
+            print(
+                "\n------------------------------------"
+                "Encountered an error while trying to remove file {file_path}.  Double"
+                "check to make sure the server is configured correctly, permissions are"
+                "okay, and everything is ship shape, then try again.")
+            return;
+
+        # Finally, mark that the file was removed
+        file.expiration = None;
+    db.session.commit()
+
+    print(f"\nDone!  {files_removed} file(s) removed")
+
+""" For a file of a given size, determine the largest allowed lifespan of that file
+
+Based on the current app's configuration:  Specifically, the MAX_CONTENT_LENGTH, as well
+as FHOST_{MIN,MAX}_EXPIRATION.
+
+This lifespan may be shortened by a user's request, but no files should be allowed to
+expire at a point after this number.
+
+Value returned is a duration in milliseconds.
+"""
+def get_max_lifespan(filesize: int) -> int:
+    min_exp = app.config.get("FHOST_MIN_EXPIRATION", 30 * 24 * 60 * 60 * 1000)
+    max_exp = app.config.get("FHOST_MAX_EXPIRATION", 365 * 24 * 60 * 60 * 1000)
+    max_size = app.config.get("MAX_CONTENT_LENGTH", 256 * 1024 * 1024)
+    return min_exp + int((-max_exp + min_exp) * (filesize / max_size - 1) ** 3)

instance/config.example.py

@@ -45,6 +45,19 @@ MAX_CONTENT_LENGTH = 256 * 1024 * 1024 # Default: 256MiB
 MAX_URL_LENGTH = 4096
 
 
+# The minimum and maximum amount of time we'll retain a file for
+#
+# Small files (nearing zero bytes) are stored for the longest possible expiration date,
+# while larger files (nearing MAX_CONTENT_LENGTH bytes) are stored for the shortest amount
+# of time.  Values between these two extremes are interpolated with an exponential curve,
+# like the one shown on the index page.
+#
+# All times are in milliseconds.  If you want all files to be stored for the same amount
+# of time, set these to the same value.
+FHOST_MIN_EXPIRATION = 30  * 24 * 60 * 60 * 1000
+FHOST_MAX_EXPIRATION = 365 * 24 * 60 * 60 * 1000
+
+
 # Use the X-SENDFILE header to speed up serving files w/ compatible webservers
 #
 # Some webservers can be configured use the X-Sendfile header to handle sending

migrations/versions/939a08e1d6e5_.py

@@ -0,0 +1,81 @@
+"""add file expirations
+
+Revision ID: 939a08e1d6e5
+Revises: 7e246705da6a
+Create Date: 2022-11-22 12:16:32.517184
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '939a08e1d6e5'
+down_revision = '7e246705da6a'
+
+from alembic import op
+from flask import current_app
+from flask_sqlalchemy import SQLAlchemy
+from pathlib import Path
+import sqlalchemy as sa
+from sqlalchemy.ext.automap import automap_base
+from sqlalchemy.orm import Session
+
+import os
+import time
+
+""" For a file of a given size, determine the largest allowed lifespan of that file
+
+Based on the current app's configuration:  Specifically, the MAX_CONTENT_LENGTH, as well
+as FHOST_{MIN,MAX}_EXPIRATION.
+
+This lifespan may be shortened by a user's request, but no files should be allowed to
+expire at a point after this number.
+
+Value returned is a duration in milliseconds.
+"""
+def get_max_lifespan(filesize: int) -> int:
+    min_exp = current_app.config.get("FHOST_MIN_EXPIRATION", 30 * 24 * 60 * 60 * 1000)
+    max_exp = current_app.config.get("FHOST_MAX_EXPIRATION", 365 * 24 * 60 * 60 * 1000)
+    max_size = current_app.config.get("MAX_CONTENT_LENGTH", 256 * 1024 * 1024)
+    return min_exp + int((-max_exp + min_exp) * (filesize / max_size - 1) ** 3)
+
+Base = automap_base()
+
+def upgrade():
+    op.add_column('file', sa.Column('expiration', sa.BigInteger()))
+
+    bind = op.get_bind()
+    Base.prepare(autoload_with=bind)
+    File = Base.classes.file
+    session = Session(bind=bind)
+
+    storage = Path(current_app.config["FHOST_STORAGE_PATH"])
+    current_time = time.time() * 1000;
+
+    # List of file hashes which have not expired yet
+    # This could get really big for some servers
+    try:
+        unexpired_files = os.listdir(storage)
+    except FileNotFoundError:
+        return # There are no currently unexpired files
+
+    # Calculate an expiration date for all existing files
+    files = session.scalars(
+            sa.select(File)
+            .where(
+                sa.not_(File.removed),
+                File.sha256.in_(unexpired_files)
+            )
+        )
+    updates = [] # We coalesce updates to the database here
+    for file in files:
+        file_path = storage / file.sha256
+        stat = os.stat(file_path)
+        max_age = get_max_lifespan(stat.st_size) # How long the file is allowed to live, in ms
+        file_birth = stat.st_mtime * 1000 # When the file was created, in ms
+        updates.append({'id': file.id, 'expiration': int(file_birth + max_age)})
+
+    # Apply coalesced updates
+    session.bulk_update_mappings(File, updates)
+    session.commit()
+
+def downgrade():
+    op.drop_column('file', 'expiration')

templates/index.html

@@ -11,6 +11,15 @@ Or you can shorten URLs:
 
 File URLs are valid for at least 30 days and up to a year (see below).
 Shortened URLs do not expire.
+
+Files can be set to expire sooner by adding an "expires" parameter (in hours)
+    curl -F'file=@yourfile.png' -F'expires=24' {{ fhost_url }}
+OR by setting "expires" to a timestamp in epoch milliseconds
+    curl -F'file=@yourfile.png' -F'expires=1681996320000' {{ fhost_url }}
+
+Expired files won't be removed immediately, but will be removed as part of
+the next purge.
+
 {% set max_size = config["MAX_CONTENT_LENGTH"]|filesizeformat(True) %}
 Maximum file size: {{ max_size }}
 Not allowed: {{ config["FHOST_MIME_BLACKLIST"]|join(", ") }}
@@ -22,7 +31,7 @@ FILE RETENTION PERIOD
 retention = min_age + (-max_age + min_age) * pow((file_size / max_size - 1), 3)
 
    days
-    365 |  \\
+ {{'{: 6}'.format(config.get("FHOST_MAX_EXPIRATION", 31536000000)//86400000)}} |  \\
         |   \\
         |    \\
         |     \\
@@ -30,7 +39,7 @@ retention = min_age + (-max_age + min_age) * pow((file_size / max_size - 1), 3)
         |       \\
         |        ..
         |          \\
-  197.5 | ----------..-------------------------------------------
+ {{'{: 6.1f}'.format((config.get("FHOST_MIN_EXPIRATION", 2592000000)/2 + config.get("FHOST_MAX_EXPIRATION", 31536000000)/2)/86400000)}} | ----------..-------------------------------------------
         |             ..
         |               \\
         |                ..
@@ -39,7 +48,7 @@ retention = min_age + (-max_age + min_age) * pow((file_size / max_size - 1), 3)
         |                       ...
         |                          ....
         |                              ......
-     30 |                                    ....................
+ {{'{: 6}'.format(config.get("FHOST_MIN_EXPIRATION", 2592000000)//86400000)}} |                                    ....................
           0{{ ((config["MAX_CONTENT_LENGTH"]/2)|filesizeformat(True)).split(" ")[0].rjust(27) }}{{ max_size.split(" ")[0].rjust(27) }}
            {{ max_size.split(" ")[1].rjust(54) }}
 </pre>