.dockerignore

@@ -0,0 +1,23 @@
+.DS_Store
+._*
+*.pyc
+__pycache__/
+.mypy_cache/
+.pytest_cache/
+.github/
+
+venv/
+.venv/
+.docker-venv/
+
+build/
+dist/
+pip_dist/
+!pip_dist/archivebox.egg-info/requires.txt
+brew_dist/
+assets/
+
+data/
+output/
+
+tmp/

.forgejo/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,16 @@
+---
+name: Feature request
+about: Suggest an improvement or new feature for the web UI
+title: ''
+labels: 'enhancement'
+assignees: ''
+
+---
+
+**Description**
+
+A clear and concise description of what you want to be implemented.
+
+**Additional Context**
+
+If applicable, please provide any extra information, external links, or screenshots that could be useful.

.gitignore

@@ -0,0 +1,136 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Files and folder
+tmp/
+*~
+
+.Cookies/
+archivist/media/

.vscode/settings.json

@@ -0,0 +1,19 @@
+{
+    "cSpell.words": [
+        "artstation",
+        "devantart",
+        "iframes",
+        "LANCZOS",
+        "ncols",
+        "nostatic",
+        "phash",
+        "Popover",
+        "preimport",
+        "runserver",
+        "SLUGIFYING",
+        "taggit",
+        "viewsets",
+        "virtualenv",
+        "whitenoise"
+    ],
+}

Dockerfile-Dev

@@ -0,0 +1,51 @@
+# Use an official Python runtime as a parent image
+#FROM python:3.10-slim-bullseye
+FROM python:3.11-alpine
+
+LABEL name="Gallery-Archivist" \
+    maintainer="Aroy-Art" \
+    description="All-in-one personal social-media/art site archiving container" \
+    homepage="https://git.aroy-art.com/Aroy/Gallery-Archivist" \
+    documentation=""
+ 
+ENV PYTHONUNBUFFERED 1
+
+# Install apk dependencies
+RUN apk update && \
+    apk add curl rustup cargo make gcc g++ automake subversion python3-dev
+
+# Create non-privileged user
+ARG USER_ID
+RUN adduser -D -u $USER_ID archivist
+#RUN addgroup -S $GROUP \
+#    && adduser --system --create-home --gid $GROUP --groups audio,video $USER
+
+# Download latest yt-dlp version and make executable
+RUN curl -L https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /usr/local/bin/yt-dlp && \
+    chmod a+rx /usr/local/bin/yt-dlp
+ 
+# Install latest stable gallery-dl
+RUN pip install -U gallery-dl
+
+COPY ./requirements.txt /app/
+
+# Set the working directory to /app
+WORKDIR /app
+
+# Copy the current directory contents into the container at /app
+#COPY . /app
+
+# Install any needed packages specified in requirements.txt
+#RUN export PYTHONPATH=/usr/bin/python && pip install --no-cache-dir -r requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Make port 8000 available to the world outside this container
+EXPOSE 8000
+
+USER archivist
+
+# Define environment variable
+#ENV DJANGO_SETTINGS_MODULE=mysite.settings.production
+
+# Run the command to start Django
+#CMD ["python", "archivist/manage.py", "runserver", "0.0.0.0:8000"]

README.md

@@ -0,0 +1,199 @@
+# Gallery-Archivist
+
+---
+[![Please don't upload to GitHub](https://nogithub.codeberg.page/badge.svg)](https://nogithub.codeberg.page)
+
+My try to make a Social media archiving web tool with django and gallery-dl
+
+## Features / Roadmap
+
+*Note!* This is still in early development so stuff **will** change.
+
+- [ ] Scraping sites primarily with gallery-dl, but also for other links found in posts.
+- [ ] Scheduled tasks
+- [ ] Site support
+  - [ ] [Furaffinity](https://www.furaffinity.net)
+  - [ ] [Twitter/X](https://twitter.com/)
+    - [x] User import With profile/banner images
+    - [ ] Import images from timeline
+- [ ] Media support and previews
+  - [x] Flash (With [Ruffle](https://github.com/ruffle-rs/ruffle/))
+  - [x] PDF (With HTML embed tag)
+  - [x] Image (With HTML img tag)
+  - [ ] Video (With [FluidPlayer](https://www.fluidplayer.com/))
+  - [ ] Compressed archive previews
+  - [ ] other text documents
+  - [ ] Easy download list of all files of post and user
+
+---
+
+## Trademarks
+
+### External Sites
+
+The logos of external sites used in Gallery-Archivist are trademarks of their respective owners. The use of these trademarks does not indicate endorsement of the trademark holder by the repository, its owners or contributors.  
+Gallery-Archivist is not endorsed by or affiliated with any of the trademark holders.
+
+---
+
+## Usage
+
+### Run without Docker
+
+- **Step 1:** Clone repo
+
+```bash
+git clone https://git.aroy-art.com/Aroy/Gallery-Archivist.git
+```
+
+- **Step 2:** Change dir to the new cloned repo
+
+```bash
+cd Gallery-Archivist
+```
+
+- **Step 3:** Make a python environment
+
+```bash
+python3 -m venv venv  
+```
+
+- **Step 4:** Activate the new python environment
+
+```bash
+source ./venv/bin/activate
+```
+
+- **Step 5:** Install python dependencies
+
+```bash
+pip install -r requirements.txt
+```
+
+- **Step 6:** Copy environment config file and change it to your liking
+
+```bash
+cp env-sample .env
+```
+
+- **Step 7:** Change directory to `./archivist`
+
+```bash
+cd ./archivist
+```
+
+- **Step 8:** Run django server
+
+```bash
+python3 manage.py runserver
+```
+
+### Manual Import Data
+
+It is possible to import data from saved gallery-dl json files
+
+- **Step 1:** Make sure that you are in the root folder of the project.
+
+- **Step 2:** Make sure to activate the python environment.
+
+```bash
+source ./venv/bin/activate
+```
+
+- **Step 3:** Change directory to `./archivist`
+
+```bash
+cd ./archivist
+```
+
+- **Step 4:** Import the json data and media file from gallery-dl  
+replace `<path>` with the path of the folder or json file to import
+
+```bash
+python manage.py import_data <path>
+```
+
+## Development
+
+### Without Docker
+
+*Note!* Instructions are made for a modern linux environment.
+
+- **Step 0:** Make sure that you have installed the dependencies.  
+You need `git, python, python-virtualenv, redis`
+
+- **Step 1:** Clone repo
+
+```bash
+git clone https://git.aroy-art.com/Aroy/Gallery-Archivist.git
+```
+
+- **Step 2:** Change dir to the new cloned repo
+
+```bash
+cd Gallery-Archivist
+```
+
+- **Step 3:** Make a python environment
+
+```bash
+python3 -m venv venv  
+```
+
+- **Step 4:** Activate the new python environment
+
+```bash
+source ./venv/bin/activate
+```
+
+- **Step 5:** Install python dependencies
+
+```bash
+pip install -r requirements.txt
+```
+
+- **Step 6:** Copy environment config file and change it to your liking
+
+```bash
+cp env-sample archivist/.env
+```
+
+- **Step 7:** Change directory to archivist
+
+```bash
+cd archivist
+```
+
+- **Step 8:** Crate and Run django database migrations
+
+```Bash
+python manage.py makemigrations
+```
+
+```bash
+python manage.py migrate
+```
+
+- **Step 9:** Start the redis server
+
+```bash
+redis-server
+```
+
+- **Step :10** Open another shell inside of `archivist/` folder and start the django server
+
+```bash
+python3 manage.py runserver
+```
+
+- **Step :11** Open another shell inside of `archivist/` folder and start the celery worker
+
+```bash
+celery -A core worker -l info
+```
+
+- **Step :12** Open another shell inside of `archivist/` folder and start the celery beat
+
+```bash
+celery -A core beat -l info
+```

archivist/apps/api/config.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class APIConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'apps.api'
+    label = 'apps_api'

archivist/apps/api/urls.py

@@ -0,0 +1,7 @@
+from django.urls import path
+
+app_name = 'api'
+
+urlpatterns = [
+    path('profile/seen_posts/<str:submission_hash>', UserProfileAPIView.seen_post, name='user_seen_post'),
+]

archivist/apps/api/views.py

@@ -0,0 +1,42 @@
+from django.http import JsonResponse
+
+# Create your views here.
+
+from django.contrib.auth.models import User, Group
+
+from apps.user.models import UserProfile, SeenPost
+
+from apps.sites.models import Submissions
+
+class UserProfileAPIView(APIView):
+    
+    def seen_post(request, submission_hash):      
+        
+        user = UserProfile.objects.get(user=request.user)
+        
+        if request.method == 'GET':            
+            try:
+                submission = Submissions.objects.get(submission_hash=submission_hash)    
+            
+                try:
+                    SeenPost.objects.get(user=user, post_id=submission.pk)
+
+                    return JsonResponse({'seen': True}, status=status.HTTP_200_OK)
+
+                except SeenPost.DoesNotExist:
+                    return JsonResponse({'seen': False}, status=status.HTTP_200_OK)
+            
+            except Submissions.DoesNotExist:
+                return JsonResponse({'message': 'Submission not found.'}, status=status.HTTP_404_NOT_FOUND)
+
+                
+        if request.method == 'PUT':
+            submission = Submissions.objects.get(submission_hash=submission_hash)
+            
+            SeenPost.objects.get_or_create(user=user, post_id=submission.pk)
+
+            return JsonResponse({'seen': True}, status=status.HTTP_200_OK)
+            
+        else:
+            return JsonResponse({'message': 'Only GET and PUT requests are allowed.'}, status=status.HTTP_405_METHOD_NOT_ALLOWED)
+

archivist/apps/authentication/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

archivist/apps/authentication/config.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class AuthConfig(AppConfig):
+    name = 'apps.auth'
+    label = 'apps_auth'

archivist/apps/authentication/forms.py

@@ -0,0 +1,55 @@
+from django import forms
+from django.contrib.auth.forms import UserCreationForm
+from django.contrib.auth.models import User
+
+
+class LoginForm(forms.Form):
+    username = forms.CharField(
+        widget=forms.TextInput(
+            attrs={
+                "placeholder": "Username",
+                "class": "form-control"
+            }
+        ))
+    password = forms.CharField(
+        widget=forms.PasswordInput(
+            attrs={
+                "placeholder": "Password",
+                "class": "form-control"
+            }
+        ))
+
+
+class SignUpForm(UserCreationForm):
+    username = forms.CharField(
+        widget=forms.TextInput(
+            attrs={
+                "placeholder": "Username",
+                "class": "form-control"
+            }
+        ))
+    email = forms.EmailField(
+        widget=forms.EmailInput(
+            attrs={
+                "placeholder": "Email",
+                "class": "form-control"
+            }
+        ))
+    password1 = forms.CharField(
+        widget=forms.PasswordInput(
+            attrs={
+                "placeholder": "Password",
+                "class": "form-control"
+            }
+        ))
+    password2 = forms.CharField(
+        widget=forms.PasswordInput(
+            attrs={
+                "placeholder": "Password check",
+                "class": "form-control"
+            }
+        ))
+
+    class Meta:
+        model = User
+        fields = ('username', 'email', 'password1', 'password2')

archivist/apps/authentication/models.py

@@ -0,0 +1,3 @@
+from django.db import models
+
+# Create your models here.

archivist/apps/authentication/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

archivist/apps/authentication/urls.py

@@ -0,0 +1,9 @@
+from django.urls import path
+from .views import login_view, register_user
+from django.contrib.auth.views import LogoutView
+
+urlpatterns = [
+    path("login/", login_view, name="login"),
+    path("register/", register_user, name="register"),
+    path("logout/", LogoutView.as_view(), name="logout"),
+]

archivist/apps/authentication/views.py

@@ -0,0 +1,68 @@
+# Create your views here.
+from django.shortcuts import render, redirect
+from django.contrib.auth import authenticate, login
+
+from apps.user.models import UserProfile  # Importing the UserProfile model
+
+from .forms import LoginForm, SignUpForm  # Importing form classes for login and signup
+
+
+# View function for user login
+def login_view(request):
+    # Check if the user is already authenticated, if so, redirect them to the home page
+    if request.user.is_authenticated:
+        next_page = request.GET.get('next', '/')  # Get the 'next' parameter from the URL, default to '/'
+        return redirect(next_page)
+    
+    form = LoginForm(request.POST or None)  # Create a login form instance
+
+    msg = None  # Initialize a message variable
+
+    if request.method == "POST":
+        if form.is_valid():
+            username = form.cleaned_data.get("username")
+            password = form.cleaned_data.get("password")
+            
+            # Authenticate user using the provided username and password
+            user = authenticate(username=username, password=password)
+            
+            if user is not None:
+                login(request, user)  # Log in the authenticated user
+                next_page = request.GET.get('next', '/')  # Get the 'next' parameter from the URL
+                return redirect(next_page)  # Redirect to the 'next' page after successful login
+            else:
+                msg = 'Invalid credentials'  # Set error message for invalid credentials
+        else:
+            msg = 'Error validating the form'  # Set error message for form validation error
+
+    return render(request, "accounts/login.html", {"form": form, "msg": msg})
+
+
+# View function for user registration
+def register_user(request):
+    msg = None  # Initialize a message variable
+    success = False  # Initialize a success flag
+
+    if request.method == "POST":
+        form = SignUpForm(request.POST)   # Create a signup form instance
+        
+        if form.is_valid():
+            form.save()  # Save the user details from the form
+            username = form.cleaned_data.get("username")
+            raw_password = form.cleaned_data.get("password1")
+            
+            # Authenticate the newly registered user
+            user = authenticate(username=username, password=raw_password)
+            
+            # Create a UserProfile instance associated with the registered user
+            profile = UserProfile(user=user)
+            profile.save()
+
+            msg = 'User created - please <a href="/login">login</a>.'  # Set success message with a login link
+            success = True  # Set success flag to True
+        else:
+            msg = 'Form is not valid'  # Set error message for invalid form data
+    else:
+        form = SignUpForm()  # Create an empty signup form instance for GET requests
+
+    return render(request, "accounts/register.html", {"form": form, "msg": msg, "success": success})

archivist/apps/config.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class AppsConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'apps'
+    label = 'apps'

archivist/apps/context_processors.py

@@ -0,0 +1,4 @@
+from django.conf import settings
+
+def debug(context):
+  return {'DEBUG': settings.DEBUG}

archivist/apps/files/admin.py

@@ -0,0 +1,63 @@
+from django.contrib import admin
+from django import forms
+from django.urls import reverse
+from django.utils.html import format_html
+
+from .models import Metadata_Files, Submission_File, User_Profile_Images, User_Banner_Images
+
+# Register your models here.
+
+class Submission_FileAdmin(admin.ModelAdmin):
+    list_display = ('id', 'file_link', 'date_added', 'file_hash', 'file',)
+    search_fields = ['file_name', 'file_hash']
+
+    def file_link(self, obj):
+        url = reverse("admin:files_submission_file_change", args=[obj.id])
+        return format_html('<a href="{}">{}</a>', url, obj.file_name)
+    
+    file_link.short_description = 'File Name'
+
+
+class User_Banner_ImagesAdmin(admin.ModelAdmin):
+    list_display = ('id', 'file_link', 'date_added', 'file_hash', 'file',)
+
+    def file_link(self, obj):
+        url = reverse("admin:files_user_banner_images_change", args=[obj.id])
+        return format_html('<a href="{}">{}</a>', url, obj.file_name)
+    
+    file_link.short_description = 'File Name'
+
+
+class User_Profile_ImagesAdmin(admin.ModelAdmin):
+    list_display = ('id', 'file_link', 'date_added', 'file_hash', 'file',)
+
+    def get_form(self, request, obj=None, **kwargs):
+       form = super().get_form(request, obj, **kwargs)
+       form.base_fields['file_hash'].widget = forms.TextInput(attrs={'readonly': True})
+       return form
+
+    def file_link(self, obj):
+        url = reverse("admin:files_user_profile_images_change", args=[obj.id])
+        return format_html('<a href="{}">{}</a>', url, obj.file_name)
+    
+    file_link.short_description = 'File Name'
+
+class Metadata_FilesAdmin(admin.ModelAdmin):
+    list_display = ('id', 'file_link', 'date_added', 'file_hash', 'file',)
+
+    def get_form(self, request, obj=None, **kwargs):
+       form = super().get_form(request, obj, **kwargs)
+       form.base_fields['file_hash'].widget = forms.TextInput(attrs={'readonly': True})
+       return form
+
+    def file_link(self, obj):
+        url = reverse("admin:files_metadata_files_change", args=[obj.id])
+        return format_html('<a href="{}">{}</a>', url, obj.file_name)
+    
+    file_link.short_description = 'File Name'
+
+
+admin.site.register(Metadata_Files, Metadata_FilesAdmin)
+admin.site.register(Submission_File, Submission_FileAdmin)
+admin.site.register(User_Banner_Images, User_Banner_ImagesAdmin)
+admin.site.register(User_Profile_Images, User_Profile_ImagesAdmin)

archivist/apps/files/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class FilesConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'apps.files'

archivist/apps/files/config.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class FilesConfig(AppConfig):
+    name = 'apps.files'
+    label = 'apps_files'

archivist/apps/files/forms.py

@@ -0,0 +1,7 @@
+from django import forms
+from .models import Submission_File
+
+class UploadFileForm(forms.ModelForm):
+    class Meta:
+        model = Submission_File
+        fields = ['file']

archivist/apps/files/models.py

@@ -0,0 +1,111 @@
+from django.db import models
+from django.urls import reverse
+
+from django.utils.translation import gettext_lazy as _
+
+# Create your models here.
+
+def get_upload_to(instance, filename, folder):
+    return f'{folder}/{instance.file_hash[:2]}/{filename}'
+
+def get_upload_to_metadata(instance, filename):
+    return get_upload_to(instance, filename, 'metadata')
+
+def get_upload_to_submission(instance, filename):
+    return get_upload_to(instance, filename, 'submissions')
+
+def get_upload_to_profile(instance, filename):
+    return get_upload_to(instance, filename, 'profiles')
+
+def get_upload_to_banner(instance, filename):
+    return get_upload_to(instance, filename, 'banners')
+
+
+class User_Profile_Images(models.Model):
+
+    file_hash  = models.CharField(unique=True, max_length=64)
+    file_name  = models.CharField(max_length=150, blank=True)
+    file       = models.FileField(upload_to=get_upload_to_profile, blank=True)
+    file_mime  = models.CharField(max_length=64, blank=True)
+    file_ext   = models.CharField(max_length=64, blank=True)
+    size       = models.PositiveIntegerField(null=True)
+    date_added = models.DateTimeField(auto_now_add=True, editable=True)
+
+    class Meta:
+        verbose_name = _("User Profile Image")
+        verbose_name_plural = _("User Profile Images")
+
+    def __str__(self):
+        return self.file_name
+
+    def get_absolute_url(self):
+        return reverse('files:serve_content_file', args=['user_profile', self.file_hash])
+
+
+class User_Banner_Images(models.Model):
+
+    file_hash  = models.CharField(unique=True, max_length=64)
+    file_name  = models.CharField(max_length=150, blank=True)
+    file       = models.FileField(upload_to=get_upload_to_banner, blank=True)
+    file_mime  = models.CharField(max_length=64, blank=True)
+    file_ext   = models.CharField(max_length=64, blank=True)
+    size       = models.PositiveIntegerField(null=True)
+    date_added = models.DateTimeField(auto_now_add=True, editable=True)
+
+    class Meta:
+        verbose_name = _("User Banner Image")
+        verbose_name_plural = _("User Banner Images")
+
+    def __str__(self):
+        return self.file_name
+    
+    def get_absolute_url(self):
+        return reverse('files:serve_content_file', args=['user_banner', self.file_hash])
+
+
+class Submission_File(models.Model):
+
+    file_hash    = models.CharField(unique=True, max_length=64)
+    file_name    = models.CharField(max_length=150, blank=True)
+    file         = models.FileField(upload_to=get_upload_to_submission, blank=True)
+    file_mime    = models.CharField(max_length=64, blank=True)
+    file_ext     = models.CharField(max_length=64, blank=True)
+    size         = models.PositiveIntegerField(null=True)
+    date_added   = models.DateTimeField(auto_now_add=True, editable=True)
+
+    extra_file   = models.BooleanField(default=False)
+    file_source  = models.CharField(max_length=32, blank=True)
+
+    image_height = models.PositiveIntegerField(null=True)
+    image_width  = models.PositiveIntegerField(null=True)
+
+    class Meta:
+        verbose_name = _("Submission File")
+        verbose_name_plural = _("Submission Files")
+
+    def __str__(self):
+        return self.file_name
+
+    def get_absolute_url(self):
+        return reverse('files:serve_content_file', args=['submission', self.file_hash])
+
+
+class Metadata_Files(models.Model):
+
+    file_hash  = models.CharField(unique=True, max_length=64)
+    file_name  = models.CharField(max_length=150, blank=True)
+    file       = models.FileField(upload_to=get_upload_to_metadata, blank=True)
+    file_mime  = models.CharField(max_length=64, blank=True)
+    file_ext   = models.CharField(max_length=64, blank=True)
+    size       = models.PositiveIntegerField(null=True)
+    date_added = models.DateTimeField(auto_now_add=True, editable=True)
+
+    class Meta:
+        verbose_name = _("Metadata File")
+        verbose_name_plural = _("Metadata Files")
+
+    def __str__(self):
+        return self.file_name
+    
+    def get_absolute_url(self):
+        return reverse('files:serve_content_file', args=['metadata', self.file_hash])

archivist/apps/files/tasks.py

@@ -0,0 +1,39 @@
+import os
+import zipfile
+import json
+from celery import shared_task
+
+@shared_task
+def list_zip_contents(zip_path):
+    """List the contents of a ZIP file.
+
+    Args:
+        zip_path (str): The path to the ZIP file.
+
+    Returns:
+        str: A JSON string containing the list of files in the ZIP archive
+        or an error/warning message if applicable.
+    """
+    try:
+        # Check if the ZIP file size exceeds 2 GiB
+        if os.path.getsize(zip_path) >= 2 * 1024 * 1024 * 1024:
+            return json.dumps({"warning": "archive file is too big (>2GiB), ignoring"})
+        else:
+            # Open the ZIP file
+            with zipfile.ZipFile(zip_path, 'r') as zip_ref:
+                file_list = []
+                # Iterate over each file in the ZIP archive
+                for info in zip_ref.infolist():
+                    # Append file details to the list
+                    file_list.append({
+                        "name": info.filename,
+                        "size": info.file_size,
+                        "date": info.date_time,
+                        "crc": info.CRC,
+                        "compressed_size": info.compress_size,
+                    })
+                # Return the list of files as a JSON string
+                return json.dumps({"files": file_list})
+    except Exception as e:
+        # Return an error message if an exception occurs
+        return json.dumps({"error": str(e)})

archivist/apps/files/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

archivist/apps/files/urls.py

@@ -0,0 +1,11 @@
+from django.urls import re_path, path
+from .views import serve_content_file, fileUpload
+
+app_name = "files"
+
+urlpatterns = [
+    # Add a URL pattern that captures the file path
+    path('<folder>/<str:file_hash>', serve_content_file, name='serve_content_file'),
+    # Other URL patterns if any
+    path('upload/', fileUpload, name='file_upload'),
+]

archivist/apps/files/views.py

@@ -0,0 +1,105 @@
+# Create your views here.
+
+from django.contrib.auth.decorators import login_required
+from django.http import FileResponse, HttpResponse, HttpResponseRedirect
+from django.shortcuts import get_object_or_404, render
+
+import os
+from blake3 import blake3
+
+from .forms import UploadFileForm
+from .models import  User_Banner_Images, User_Profile_Images, Metadata_Files, Submission_File
+
+
+MODEL_MAP = {
+   'user_profile': User_Profile_Images,
+   'user_banner': User_Banner_Images,
+   'submission': Submission_File,
+   'metadata': Metadata_Files,
+}
+
+def compute_file_hash(file):
+    '''
+    Compute BLAKE3 hash of the file
+    '''
+    try:
+        hasher = blake3()
+        for chunk in file.chunks(chunk_size=65536):
+            hasher.update(chunk)
+        return hasher.hexdigest()
+    except Exception as e:
+        print(f"Error computing file hash: {e}")
+        return None
+
+
+@login_required(login_url="/login/")
+def serve_content_file(request, folder, file_hash):
+   '''
+   View function to serve content files for download or inline viewing
+   '''
+   
+   ModelClass = MODEL_MAP.get(folder)
+   if ModelClass is None:
+       return HttpResponse("Invalid folder", status=404)
+
+   download = request.GET.get('d')
+
+   try:
+       obj_file = get_object_or_404(ModelClass, file_hash=file_hash)
+       file = obj_file.file.file
+       file_name = obj_file.file_name
+
+       response = FileResponse(file)
+       if download == "1":
+           response['Content-Disposition'] = f'attachment; filename="{file_name}"'
+       else:
+           response['Content-Disposition'] = f'inline; filename="{file_name}"'
+
+       return response
+
+   except Exception as e:
+       print(f"Error serving file: {e}")
+       return HttpResponse("File not found", status=404)
+
+
+@login_required(login_url="/login/")
+def fileUpload(request):
+    '''
+    View function for handling file uploads
+    '''
+    if request.method == 'POST':
+        form = UploadFileForm(request.POST, request.FILES)
+        if form.is_valid():
+            if 'file' in request.FILES:  # Check if a file has been uploaded
+                file = form.cleaned_data['file']
+
+                file_name = file.name
+                file_hash = compute_file_hash(file)
+
+                Null, file_ext = os.path.splitext(file_name)
+                hash_file_name = file_hash + file_ext
+
+                new_submission_file, created = Submission_File.objects.get_or_create(file_hash=file_hash)
+
+                new_submission_file.file_hash = file_hash
+                new_submission_file.file_name = file_name
+                new_submission_file.file.save(hash_file_name, file) 
+
+                new_submission_file.save
+
+                return HttpResponseRedirect(f"/files/submission/{file_hash}")
+
+            else:
+                # No file was uploaded, add an error message to the context
+                error_message = 'No file was uploaded.'
+                return render(request, 'files/upload.html', {'form': form, 'error_message': error_message})
+
+        else:
+            # Form is not valid, add an error message to the context
+            error_message = 'There was an error with the form.'
+            return render(request, 'files/upload.html', {'form': form, 'error_message': error_message})
+
+    else:
+        form = UploadFileForm()
+        
+    return render(request, 'files/upload.html', {'form': form})

archivist/apps/importer/admin.py

@@ -0,0 +1,9 @@
+from django.contrib import admin
+
+from .models import ImportSourceURLs
+
+
+class ImportSourceURLsAdmin(admin.ModelAdmin):
+    list_display = ('url', 'added_by_user', 'last_imported', 'source_type', 'category', 'date_added')
+
+admin.site.register(ImportSourceURLs, ImportSourceURLsAdmin)

archivist/apps/importer/config.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class ImporterConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'apps.importer'
+    label = 'apps.importer'

archivist/apps/importer/forms.py

@@ -0,0 +1,22 @@
+from django import forms
+
+from .models import ImportSourceURLs
+
+
+class ImportSourceURLsForm(forms.Form):
+    url = forms.URLField(
+        label="Add URL",
+        required=True,
+        widget=forms.TextInput(
+            attrs={"placeholder": "https://example.com", "class": "form-control"}
+        ),
+    )
+
+
+class GalleryDLConfigForm(forms.Form):
+    text = forms.CharField(
+        label="GalleryDL Config",
+        required=True,
+        widget=forms.Textarea(attrs={"class": "form-control"}),
+    )
+

archivist/apps/importer/models.py

@@ -0,0 +1,35 @@
+from django.db import models
+
+from django.utils.translation import gettext_lazy as _
+
+from apps.sites.models import Category
+
+from apps.user.models import UserProfile
+
+class ImportSourceURLs(models.Model):
+    
+    SOURCE_TYPES = (
+        ('C', 'Complete User'),
+        ('P', 'Singel Post'),
+    )
+    
+    url = models.URLField(unique=True)
+
+    category = models.ForeignKey(Category, on_delete=models.CASCADE)    
+
+    added_by_user = models.ForeignKey(UserProfile, on_delete=models.CASCADE) 
+
+    date_added = models.DateTimeField(auto_now_add=True, editable=True)
+    
+    last_imported = models.DateTimeField(editable=True, blank=True, null=True) 
+    
+    source_type = models.CharField(max_length=1, choices=SOURCE_TYPES, default=None)
+    
+    active = models.BooleanField(default=True)
+    
+    class Meta:
+        verbose_name = _("Import Source URL")
+        verbose_name_plural = _("Import Source URLs")
+
+    def __str__(self):
+        return self.url  

archivist/apps/importer/tasks.py

@@ -0,0 +1,12 @@
+import time
+
+from celery import shared_task
+
+@shared_task
+def add(x, y):
+    return x + y
+
+@shared_task
+def wait(x):
+    time.sleep(x)
+    return f"Sleeping for {x} seconds"

archivist/apps/sites/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class SitesConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'apps.sites'

archivist/apps/sites/config.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class SitesConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'apps.sites'
+    label = 'apps.sites'

archivist/apps/sites/forms.py

@@ -0,0 +1,54 @@
+from django import forms
+
+from .models import Category
+
+class SearchForm(forms.Form):
+    q = forms.CharField(
+        label='Search', 
+        max_length=100,
+        required=False,
+        widget=forms.TextInput(
+            attrs={
+                'class': 'form-control',
+                'placeholder': 'Search'
+                }
+            ),
+        )
+
+    category = forms.ModelChoiceField(
+        label="Site", 
+        queryset=Category.objects.all(), 
+        empty_label="All Sites", # Sets the name of the null option
+        required=False,
+        widget=forms.Select(
+            attrs={
+                'class': 'form-select',
+                'placeholder': 'All Sites'
+            }
+        ),
+    )
+    
+    sort = forms.ChoiceField(
+        label="Sort by", 
+        choices=[('1', 'Date'), ('2', 'Views'), ('3', 'Likes'), ('4', 'Relevance')],
+        initial='1',
+        required=False,
+        widget=forms.Select(
+            attrs={
+                'class': 'form-select',
+            }
+        )
+    )
+    
+    mature = forms.ChoiceField(
+        label="Filter by Mature", 
+        choices=[('1', 'All'),('2', 'General'), ('3', 'Mature/Adult')],
+        initial='1',
+        required=False,
+        widget=forms.RadioSelect(
+            attrs={
+                'class': 'form-check-input',
+            }
+        ),
+    )
+

archivist/apps/sites/management/commands/import_data.py

@@ -0,0 +1,559 @@
+# /management/commands/import_data.py
+
+import os
+import json
+import requests
+from blake3 import blake3
+from tqdm.auto import tqdm
+from PIL import Image
+
+from datetime import datetime
+
+from django.core.management.base import BaseCommand
+from django.core.files.base import ContentFile
+from django.utils.text import slugify
+from django.utils import timezone
+from django.core.exceptions import ObjectDoesNotExist
+
+from django.contrib.contenttypes.models import ContentType
+
+
+from apps.files.models import User_Profile_Images, User_Banner_Images, Submission_File, Metadata_Files
+
+from apps.sites.models import Category, Submissions, Users, Tags
+
+from apps.sites.furaffinity.models import FA_Submission, FA_Tags, FA_User, FA_Species, FA_Gender, FA_Mature
+from apps.sites.twitter.models import Twitter_Submissions, Twitter_Users, Twitter_Tags
+
+from utils.files import get_mime_type
+from utils.strings import get_urls
+
+
+class Command(BaseCommand):
+    help = 'Import data from JSON files in a folder or a single JSON file to the Twitter archive'
+
+    def add_arguments(self, parser):
+        parser.add_argument('path', type=str, help='Path to the folder containing JSON files or a single JSON file')
+        parser.add_argument('--delete', action='store_true', help='Delete imported files')
+
+
+    def handle(self, *args, **kwargs):
+        path = kwargs['path']
+        delete = kwargs['delete']
+
+        if os.path.isfile(path):
+            self.process_json_file(path, delete)
+        elif os.path.isdir(path):
+            self.process_json_folder(path, delete)
+        else:
+            self.stdout.write(self.style.ERROR(f"The path '{path}' is not a valid file or folder."))
+            return
+
+
+    def process_json_file(self, file_path, delete):
+        #self.stdout.write(self.style.NOTICE(f"Importing data from: {file_path}"))
+        tqdm.write(f"Importing data from: {file_path}")
+
+        with open(file_path) as f:
+            data = json.load(f)
+
+        self.import_data(data, file_path, delete)
+
+        tqdm.write(self.style.SUCCESS('Data imported successfully.'))
+
+
+    def process_json_folder(self, folder_path, delete):
+        if not os.path.exists(folder_path):
+            #self.stdout.write(self.style.ERROR(f"The folder '{folder_path}' does not exist."))
+            tqdm.write(self.style.ERROR(f"The folder '{folder_path}' does not exist."))
+            return
+
+        for root, dirs, files in tqdm(os.walk(folder_path), dynamic_ncols=True):
+            for file_name in files:
+                if file_name.endswith('.json'):
+                    file_path = os.path.join(root, file_name)
+                    self.process_json_file(file_path, delete)
+                    
+
+    def compute_file_hash(self, file_path):
+        """ Compute BLAKE3 hash of the file """
+        try:
+            hasher = blake3()
+            with open(file_path, 'rb') as f:
+                while chunk := f.read(65536):
+                    hasher.update(chunk)
+            return hasher.hexdigest()
+        except Exception as e:
+            tqdm.write(self.style.WARNING(f"Error computing file hash: {e}"))
+            return None
+    def compute_string_hash(self, string):
+        """ Compute BLAKE3 hash of the string """
+        try:
+            hasher = blake3()
+            hasher.update(string.encode())
+            return hasher.hexdigest()
+        except Exception as e:
+            tqdm.write(self.style.WARNING(f"Error computing string hash: {e}"))
+            return None
+    def import_file(self, file_path, model, delete=False):
+        """
+        Imports a file if it doesn't already exist in the database and returns the instance.
+
+        :param file_path: The path to the file to import.
+        :param model: The model class to which the file instance should be linked.
+        :param delete: Whether to delete the imported file after processing.
+        :return: The file instance.
+        """
+        
+        file_instance = None # Initialize file_instance to None
+        
+        if os.path.exists(file_path):
+            file_hash = self.compute_file_hash(file_path)
+            
+            file_name = os.path.basename(file_path)
+            Null, file_ext = os.path.splitext(file_name)
+            hash_file_name = file_hash + file_ext
+                    
+            try:
+                file_instance = model.objects.get(file_hash=file_hash)
+                
+                file_instance.file_ext = file_ext
+                file_instance.size = os.path.getsize(file_path)
+                file_instance.file_mime = get_mime_type(file_path)
+                
+                if file_instance.file_mime.startswith("image/"):
+                    im = Image.open(file_instance.file)
+                    file_instance.image_height, file_instance.image_width = im.size
+                else:
+                    file_instance.image_height = None
+                    file_instance.image_width = None
+                
+                file_instance.save()
+                
+                tqdm.write(self.style.NOTICE(f"Skipping: {file_path} file, already imported"))
+                
+            except model.DoesNotExist:
+                # If the file doesn't exist, create a new file instance
+                with open(file_path, 'rb') as file:
+                    file_instance = model()
+                    file_instance.file_hash = file_hash
+
+                    file_instance.file.save(hash_file_name, file)
+
+                    file_instance.file_ext = file_ext
+                    file_instance.file_mime = get_mime_type(file_path)
+                    file_instance.size = os.path.getsize(file_path)
+                    
+                    if file_instance.file_mime.startswith("image/"):
+                        im = Image.open(file_instance.file)
+                        file_instance.image_height, file_instance.image_width = im.size
+                    else:
+                        file_instance.image_height = None
+                        file_instance.image_width = None
+                    
+                    file_instance.file_name = file_name
+                    file_instance.save()
+
+                    tqdm.write(self.style.NOTICE(f"Import file: {file_path}"))
+
+            if delete:
+                self.delete_imported_file(file_path)
+
+        return file_instance
+
+
+    def delete_imported_file(self, file_path, delete=False):
+        """
+        Delete the file if the --delete flag is used
+        
+        :param delete: Whether to delete the imported file after processing.
+        """
+        if delete:
+            if os.path.exists(file_path):
+                os.remove(file_path)
+                tqdm.write(self.style.SUCCESS(f"Deleted: {file_path}"))
+            else:
+                tqdm.write(self.style.WARNING(f"File not found: {file_path}"))
+
+
+    def import_data(self, data, json_file_path, delete):    
+
+        category = data['category']
+
+        if category == "twitter":
+            self.import_from_twitter(data, json_file_path, delete)
+        
+        elif category == "furaffinity":
+            self.import_from_furaffinity(data, json_file_path, delete)
+        
+        else:
+            tqdm.write(f"Skipping '{category}' not implemented")
+        
+    def import_twitter_user(self, data, file_path, category, delete=False):
+        """
+        Import a Twitter user from the provided data into the database.
+
+        Parameters:
+            data (dict): The data containing information about the Twitter user.
+            file_path (str): The file path for importing user images.
+            delete (bool): Flag indicating whether to delete user images after importing it.
+
+        Returns:
+            Twitter_Users: The Twitter user object imported or retrieved from the database.
+        """
+
+        content_type = ContentType.objects.get_for_model(Twitter_Users)
+
+        author, created = Twitter_Users.objects.get_or_create(artist_id=data['author']['id'])
+        
+        author.artist           = data['author']['nick']
+        author.artist_url       = data['author']['name']
+        author.date             = timezone.make_aware(datetime.strptime(data['author']["date"], "%Y-%m-%d %H:%M:%S"))
+        author.description      = data['author']['description']
+        if 'url' in data['author'].keys():
+            author.extra_url        = data['author']['url']
+        author.location         = data['author']['location']
+        author.verified         = data['author']['verified']
+        
+        if author.favourites_count == None or data['author']["favourites_count"] > author.favourites_count:
+            author.favourites_count = data['author']["favourites_count"]
+        if author.followers_count == None or data['author']["followers_count"] > author.followers_count:
+            author.followers_count = data['author']["followers_count"]
+        if author.friends_count == None or data['author']["friends_count"] > author.friends_count:
+            author.friends_count = data['author']["friends_count"]
+        if author.media_count == None or data['author']["media_count"] > author.media_count:
+            author.media_count = data['author']["media_count"]
+        if author.listed_count == None or data['author']["listed_count"] > author.listed_count:
+            author.listed_count = data['author']["listed_count"]
+        if author.statuses_count == None or data['author']["statuses_count"] > author.statuses_count:
+            author.statuses_count = data['author']["statuses_count"]
+        
+        if data['subcategory'] == "avatar":
+            author.profile_image = data['author']['profile_image']
+        
+            author.icon = self.import_file(file_path, User_Profile_Images, delete)
+        
+        elif data['subcategory'] == "background":
+            
+            author.profile_banner = data['author']['profile_banner']
+        
+            author.banner = self.import_file(file_path, User_Banner_Images, delete)
+        
+        
+        author_hash = self.compute_string_hash(data['author']['name'] + data['category'])
+        
+        site_user, created = Users.objects.get_or_create(user_hash=author_hash)
+        
+        site_user.category = category
+        
+
+        # Get the primary key of the twitter_submission instance
+        site_user_id = author.pk
+        
+        # Create the SubmissionsLink instance
+        site_user.content_type=content_type
+        site_user.object_id=site_user_id
+        
+        site_user.save()
+        
+        author.save()
+
+        return author, site_user
+
+
+    def import_twitter_tags(self, data: dict, category: str) -> list[Twitter_Tags]:
+        """
+        Import a Twitter tag from the provided data into the database.
+
+        Parameters:
+            data (dict): The data containing information about the Twitter tag.
+
+        Returns:
+            list[Twitter_Tags]: A list of imported or retrieved Twitter tag objects.
+        """
+        
+        content_type = ContentType.objects.get_for_model(Twitter_Tags)
+        
+        tags: list[Twitter_Tags] = []
+        
+        if "hashtags" in data:
+            for t_tag_name in data["hashtags"]:
+                    t_tag_slug = slugify(t_tag_name)
+                    try:
+                        # Check if the tag already exists in the database by name
+                        tag: Twitter_Tags = Twitter_Tags.objects.get(tag_slug=t_tag_slug)
+                        
+                        tag_id = tag.pk
+                        
+                    except ObjectDoesNotExist:
+                        # If the tag does not exist, create a new tag and generate the slug
+                        tag = Twitter_Tags(tag=t_tag_name)
+                        tag.tag_slug = t_tag_slug
+                        
+                        tag_id = tag.pk
+                        
+                    site_tags, created = Tags.objects.get_or_create(tag_slug=t_tag_slug)
+        
+                    site_tags.category.add(category)
+                    
+                    site_tags.content_type=content_type
+                    site_tags.object_id=tag_id
+    
+                    site_tags.save()
+
+                    tag.save()  # Save the tag (either new or existing)
+
+                    tags.append(tag)
+                
+        return tags
+
+
+    def import_from_twitter(self, data, json_file_path, delete):
+        
+        category, created = Category.objects.get_or_create(name=data['category'])
+        
+        category.save()
+        
+        twitter_submission, created = Twitter_Submissions.objects.get_or_create(submission_id=data["tweet_id"])    
+    
+        file_path = json_file_path.removesuffix(".json")
+        
+        # Handle author import
+        author, site_user = self.import_twitter_user(data, file_path, category, delete)        
+        
+        twitter_submission.author = author
+        
+        # Handle tag import
+        tags = self.import_twitter_tags(data, category)
+            
+        for tag in tags:
+            twitter_submission.tags.add(tag)  # Add the tag to the submission
+        
+        twitter_submission.gallery_type = data['subcategory']
+        
+        # Handle file import
+        twitter_submission.files.add(self.import_file(file_path, Submission_File, delete))
+           
+        # Handle metadata file import        
+        twitter_submission.metadata.add(self.import_file(json_file_path, Metadata_Files, delete))
+            
+        twitter_submission.description = data['content']
+
+        twitter_submission.date = timezone.make_aware(datetime.strptime(data['date'], "%Y-%m-%d %H:%M:%S"))
+        
+        twitter_submission.origin_site = data['category']        
+        
+        twitter_submission.file_extension = data['extension']
+        
+        twitter_submission.origin_filename = data['filename']
+        
+        if twitter_submission.media_num is None or data['num'] > twitter_submission.media_num:
+            twitter_submission.media_num = data['num']
+        
+        if "height" in data.keys():
+            twitter_submission.image_height = data['height']
+        if "width" in data.keys():
+            twitter_submission.image_width = data['width']
+        
+        if "sensitive" in data.keys():
+            twitter_submission.sensitive = data['sensitive']
+            
+        if "favorite_count" in data.keys():
+            twitter_submission.favorites_count = data['favorite_count']
+        
+        if "quote_count" in data.keys():
+            twitter_submission.quote_count = data['quote_count']
+        
+        if "reply_count" in data.keys():
+            twitter_submission.reply_count = data['reply_count']
+        
+        if "retweet_count" in data.keys():
+            twitter_submission.retweet_count = data['retweet_count']
+        
+        twitter_submission.lang = data['lang']
+        
+        twitter_submission.save()
+        
+        submission_hash = self.compute_string_hash(category.name + data['author']['name'] + str(data["tweet_id"]))
+        
+        submission, created = Submissions.objects.get_or_create(submission_hash=submission_hash)
+        
+        submission.category = category
+        
+        submission.author = site_user
+        
+        if twitter_submission.sensitive is not None:
+            submission.mature = twitter_submission.sensitive
+        else:
+            submission.mature = False 
+        
+        submission.date = timezone.make_aware(datetime.strptime(data['date'], "%Y-%m-%d %H:%M:%S"))
+        
+        content_type = ContentType.objects.get_for_model(Twitter_Submissions)
+
+        # Get the primary key of the twitter_submission instance
+        twitter_submission_id = twitter_submission.pk
+        
+        # Create the SubmissionsLink instance
+        submission.content_type=content_type
+        submission.object_id=twitter_submission_id
+        
+        submission.save()
+        
+        self.delete_imported_file(json_file_path, delete)
+        self.delete_imported_file(file_path, delete)
+
+
+    def import_furaffinity_user(self, data, json_file_path, category, delete):
+        content_type = ContentType.objects.get_for_model(FA_User)
+
+        artist, created = FA_User.objects.get_or_create(artist_url=data["artist_url"], artist=data["artist"])
+        
+        author_hash = self.compute_string_hash(data["artist_url"] + data['category'])
+        
+        site_user, created = Users.objects.get_or_create(user_hash=author_hash)
+        
+        site_user.category = category
+        
+        
+        # Get the primary key of the furaffinity_submission instance
+        site_user_id = artist.pk
+        
+        # Create the SubmissionsLink instance
+        site_user.content_type=content_type
+        site_user.object_id=site_user_id
+        
+        site_user.save()
+        
+        return artist, site_user
+
+
+    def import_furaffinity_tags(self, data, category):
+        
+        content_type = ContentType.objects.get_for_model(FA_Tags)
+        
+        tags: list[FA_Tags] = []
+        
+        site_tags: list[Tags] = []
+        
+        if "tags" in data:
+            for t_tag_name in data["tags"]:
+                    t_tag_slug = slugify(t_tag_name)
+                    try:
+                        # Check if the tag already exists in the database by name
+                        tag: FA_Tags = FA_Tags.objects.get(tag_slug=t_tag_slug)
+                        
+                        tag_id = tag.pk
+                        
+                    except ObjectDoesNotExist:
+                        # If the tag does not exist, create a new tag and generate the slug
+                        tag = FA_Tags(tag=t_tag_name)
+                        tag.tag_slug = t_tag_slug
+                        
+                        tag_id = tag.pk
+                        
+                    site_tag, created = Tags.objects.get_or_create(tag_slug=t_tag_slug)
+        
+                    site_tag.category.add(category)
+                    
+                    site_tag.content_type=content_type
+                    site_tag.object_id=tag_id
+    
+                    site_tag.save()
+
+                    tag.save()  # Save the tag (either new or existing)
+
+                    tags.append(tag)
+                    
+                    site_tags.append(site_tag)
+                
+        return tags, site_tags
+    
+
+    def import_from_furaffinity(self, data, json_file_path, delete):
+        
+        category, created = Category.objects.get_or_create(name=data['category'])
+        
+        category.save()
+        
+        furaffinity_submission, created = FA_Submission.objects.get_or_create(submission_id=data["id"])    
+    
+        furaffinity_submission.media_url = data["url"]
+        furaffinity_submission.title = data["title"]
+        furaffinity_submission.description = data["description"]
+        
+        furaffinity_submission.date = timezone.make_aware(datetime.strptime(data["date"], "%Y-%m-%d %H:%M:%S"))
+        
+        file_path = json_file_path.removesuffix(".json")
+        
+        # Handle author import
+        author, site_user = self.import_furaffinity_user(data, file_path, category, delete)        
+        
+        furaffinity_submission.artist = author
+        
+        # Handle tag import
+        tags, site_tags = self.import_furaffinity_tags(data, category)
+            
+        for tag in tags:
+            furaffinity_submission.tags.add(tag)  # Add the tag to the submission
+        
+        species, created = FA_Species.objects.get_or_create(species=data["species"])
+        furaffinity_submission.species = species
+        
+        # Handle mature rating import
+        mature, created = FA_Mature.objects.get_or_create(mature=data["rating"])
+        furaffinity_submission.mature_rating = mature
+        
+        furaffinity_submission.number_of_comments = data["comments"]
+        furaffinity_submission.views = data["views"]
+        
+        gender, created = FA_Gender.objects.get_or_create(gender=data["gender"])
+        furaffinity_submission.gender = gender
+        
+        furaffinity_submission.fa_theme = data["theme"]
+        furaffinity_submission.fa_category = data["fa_category"]
+        furaffinity_submission.gallery_type = data["subcategory"]
+        furaffinity_submission.file_extension = data["extension"]
+        furaffinity_submission.image_height = data["height"]
+        furaffinity_submission.image_width = data["width"]
+        
+        # Handle file import
+        furaffinity_submission.files.add(self.import_file(file_path, Submission_File, delete))
+           
+        # Handle metadata file import        
+        furaffinity_submission.metadata.add(self.import_file(json_file_path, Metadata_Files, delete))
+        
+        furaffinity_submission.save()
+            
+        submission_hash = self.compute_string_hash(category.name + data["artist_url"] + str(data["id"]))
+        
+        submission, created = Submissions.objects.get_or_create(submission_hash=submission_hash)
+        
+        submission.category = category
+        
+        submission.tags.add(*site_tags)
+        
+        submission.author = site_user
+        
+        if furaffinity_submission.mature_rating.mature != "General" and not None:
+            print("Mature")
+            submission.mature = True
+        else:
+            submission.mature = False
+        
+        submission.date = timezone.make_aware(datetime.strptime(data['date'], "%Y-%m-%d %H:%M:%S"))
+        
+        content_type = ContentType.objects.get_for_model(FA_Submission)
+
+        # Get the primary key of the twitter_submission instance
+        furaffinity_submission_id = furaffinity_submission.pk
+        
+        # Create the SubmissionsLink instance
+        submission.content_type=content_type
+        submission.object_id=furaffinity_submission_id
+        
+        submission.save()
+        
+        self.delete_imported_file(json_file_path, delete)
+        self.delete_imported_file(file_path, delete)
+        

archivist/apps/sites/templatetags/media_filters.py

@@ -0,0 +1,55 @@
+from django import template
+
+register = template.Library()
+
+@register.filter
+def is_image (mime_type):
+    '''
+    A function that takes the mime type as input and returns true if it is an image
+    '''
+    return mime_type.startswith('image/')
+
+
+@register.filter
+def is_video (mime_type):
+    '''
+    A function that takes the mime type as input and returns true if it is an video
+    '''
+    return mime_type.startswith('video/')
+
+
+@register.filter
+def is_flash (mime_type):
+    '''
+    A function that takes the mime type as input and returns true if it is an flash
+    '''
+
+    valid_flash_mime_types = [
+        'application/vnd.adobe.flash.movie',
+        'application/x-shockwave-flash',
+        'application/futuresplash',
+        'application/x-swf',
+    ]
+    
+    for valid_type in valid_flash_mime_types:
+        if valid_type in mime_type:
+            return True
+    return False
+
+
+@register.filter
+def is_pdf(mime_type):
+    """
+    A function that takes the mime type as input and returns true if it is an pdf
+    """
+    valid_pdf_mime_types = [
+        "application/pdf",
+        "application/vnd.cups-pdf",
+        "application/x-pdf",
+    ]
+
+    for valid_type in valid_pdf_mime_types:
+        if valid_type in mime_type:
+            return True
+
+    return False

archivist/apps/sites/templatetags/string_helper.py

@@ -0,0 +1,24 @@
+from django import template
+
+from utils.strings import (
+    convert_size,
+    aTag_urls,
+
+register = template.Library()
+
+@register.filter
+def size_to_human_readable (size):
+    """
+    A filter that converts the given size to a human-readable format using the utils.strings.convert_size function.
+    
+    Parameters:
+        size: The size to be converted.
+        
+    Returns:
+        The human-readable size format.
+    """
+    return convert_size(size)
+
+@register.filter
+def clickable_urls(string):
+    return aTag_urls(string)

archivist/apps/templates/404.html

@@ -0,0 +1,67 @@
+{% extends "layouts/base-electric.html" %}
+
+{% load static %}
+
+{% block title %} 404 Page Not Found {% endblock title %}
+
+{% block stylesheets %}
+<style>
+    .e-container-border{
+        padding: 4px;
+        margin: auto;
+        background-color:#222222;
+        border-radius: 25px;
+        background: linear-gradient(180deg, #4b8fca, #e73c7e, #23a6d5, #23d5ab);
+        background-size: 400% 400%;
+        animation: gradient 15s ease infinite;
+        box-shadow: 0px 2px 10px 0px #221133;
+        transition-timing-function: ease-out;
+        transition-duration: 0.3s;
+        display: flex;
+    }
+    .e-container{
+        overflow:hidden;
+        background-color:#222222;
+        padding:24px;
+        text-align:justify;
+        border-radius: 25px;
+    }
+    
+    {% comment %}
+    .containcenter{
+        margin:auto;
+    }
+    .index{
+        transform: scale(0.95);
+        transition-timing-function: ease-out;
+        transition-duration: 0.3s;
+        filter: saturate(0) contrast(75%) brightness(0.8);
+    }
+    .index:hover{
+        transform: scale(1);
+        transition-timing-function: ease-out;
+        transition-duration: 0.1s;
+        filter: saturate(1) contrast(100%) brightness(1);
+    } 
+    
+    {% endcomment %}
+</style>
+{% endblock stylesheets %}
+    
+{% block content %}
+    
+    {% include "includes/navigation-transparent.html" %}
+
+    <div class="container"> 
+        <div class="e-container-border row mb-3" tabindex="1">
+            <div class="e-container">
+                <div class="my-20 text-center">
+                    <h1 class="bold glitch" data-text="404">404</h1>
+                    <h2>Page Not Found</h2>
+                    <p>Sorry cant find that page :(</p>
+                </div>
+            </div>
+        </div>
+    </div>
+    
+{% endblock content %}

archivist/apps/templates/accounts/login.html

@@ -0,0 +1,82 @@
+{% extends "layouts/base-fullscreen.html" %}
+
+{% load static %}
+
+{% block title %} Sign In {% endblock title %} 
+
+<!-- Specific Page CSS goes HERE  -->
+{% block stylesheets %}{% endblock stylesheets %}
+
+{% block body_class %}{% endblock body_class %}
+
+{% block content %}
+
+    {% include "includes/navigation-transparent.html" %}
+
+    <style type="text/css">
+        .body-p{
+            background-image: url('{% static "/img/bg/login-bg.jpg" %}');
+            background-repeat: no-repeat;
+            background-attachment: fixed;
+            background-position: center; 
+            background-size: cover;
+        }
+
+        .div-p{
+            max-width:700px;
+            background: rgb(0,0,0);
+            background: -moz-linear-gradient(90deg, rgba(0,0,0,1) 0%, rgba(0,0,0,0.6404762588629201) 35%, rgba(255,255,255,0) 100%);
+            background: -webkit-linear-gradient(90deg, rgba(0,0,0,1) 0%, rgba(0,0,0,0.6404762588629201) 35%, rgba(255,255,255,0) 100%);
+            background: linear-gradient(90deg, rgba(0,0,0,1) 0%, rgba(0,0,0,0.6404762588629201) 35%, rgba(255,255,255,0) 100%);
+            filter: progid:DXImageTransform.Microsoft.gradient(startColorstr="#000000",endColorstr="#ffffff",GradientType=1); 
+        }
+    </style>
+    
+    <div class="d-flex flex-column body-p vh-100" loading="lazy">
+        <div class="d-flex flex-grow-1 div-p">
+            <form class="p-3 bg-body m-auto border border-2 border-info-subtle rounded bg-opacity-75" style="width: 24rem;" role="form" method="post" action="">
+                
+                {% csrf_token %}
+
+                <h3 class="fw-normal mb-3 pb-3" style="letter-spacing: 1px;">
+                    <i class="nf nf-md-login"></i> Log in
+                </h3>
+                
+                <div class="row mb-3">
+                    <p class="mb-0 text-danger text-center">
+                        {% if msg %}
+                            {{ msg | safe }}
+                        {% else %}
+                            Input your credentials
+                        {% endif %}
+                    </p>
+                </div>
+                <div class="input-group input-group-outline mb-3">
+                    {{ form.username }}
+                </div>
+                
+                <div class="input-group input-group-outline mb-3">
+                    {{ form.password }}
+                </div>
+                <div class="form-check form-switch d-flex align-items-center mb-3">
+                    <input class="form-check-input" type="checkbox" id="rememberMe">
+                    <label class="form-check-label mb-0 ms-2" for="rememberMe">Remember me</label>
+                </div>
+                <div class="text-center">
+                    <button type="submit" name="login" 
+                        class="btn btn-outline-primary bg-gradient-primary w-100 my-4 mb-2 text-light">Sign in</button>
+                </div>
+                <p class="mt-4 text-sm text-center">
+                    Don't have an account?
+                    <a href="{% url 'register' %}" class="text-primary text-gradient font-weight-bold">Sign up</a>
+                </p>
+            </form>
+            
+        </div>
+        {% include "includes/footer-auth.html" %} 
+    </div>
+
+{% endblock content %}
+
+<!-- Specific Page JS goes HERE  -->
+{% block javascripts %}{% endblock javascripts %}

archivist/apps/templates/accounts/profile.html

@@ -0,0 +1,69 @@
+{% extends "layouts/base-electric.html" %}
+
+{% load static %}
+
+{% block title %} Sites {% endblock title %}
+
+{% block stylesheets %}{% endblock stylesheets %}
+
+{% block content %}
+    
+    {% include "includes/navigation-transparent.html" %}
+    
+    <div class="container-fluid"> 
+        <div class="e-container-border e-container-radius row mb-3" tabindex="1">
+            <div class="e-container e-container-radius p-3">
+                
+                <h1 class="text-center pb-3">Profile Info</h1>
+
+                <div class="table-responsive rounded-2">
+                    <table class="table table-sm table-bordered border-primary-subtle table-striped table-hover">
+                        <tbody>
+                            <tr>
+                                <th scope="row" class="text-center">Last Login</th>
+                                <td class="text-center">{{ user.last_login }}</td>
+                            </tr>
+                            <tr>
+                                <th scope="row" class="text-center">Registration Date</th>
+                                <td class="text-center">{{ user.date_joined }}</td>
+                            </tr>
+                            <tr>
+                                <th scope="row" class="text-center">Admin Status</th>
+                                <td class="text-center">
+                                    {% if user.is_staff %}
+                                        <span class="badge bg-success">Yes</span>
+                                    {% else %}
+                                        <span class="badge bg-danger">No</span>
+                                    {% endif %}
+                                </td>
+                            </tr>
+                        </tbody>
+                    </table>
+                </div>
+                
+            </div>
+        </div>
+
+        <div class="e-container-border e-container-radius row my-3" tabindex="1">
+            <div class="e-container e-container-radius">
+                
+                <h1 class="text-center">Profile Settings</h1>
+
+                <h3>Edit your profile</h3>
+
+                <form method="post">
+                    {% csrf_token %}
+                    {{ user_form.as_p }}
+                    {{ profile_form.as_p }}
+                    <button type="submit">Save</button>
+                </form>
+                
+            </div>
+        </div>
+
+    </div>
+        
+    
+{% endblock content %}
+    
+    

archivist/apps/templates/accounts/register.html

@@ -0,0 +1,101 @@
+{% extends "layouts/base-fullscreen.html" %}
+
+{% load static %}
+
+{% block title %} Sign up {% endblock %} 
+
+{% block body_class %}{% endblock %}
+
+<!-- Specific Page CSS goes HERE  -->
+{% block stylesheets %}{% endblock stylesheets %}
+
+{% block content %}
+
+{% include 'includes/navigation-transparent.html' %} 
+<style type="text/css">
+    .body-p{
+        background-image: url('{% static "/img/bg/login-bg.jpg" %}');
+        background-repeat: no-repeat;
+        background-attachment: fixed;
+        background-position: center; 
+        background-size: cover;
+    }
+    
+    .div-p{
+        max-width:700px;
+        background: rgb(0,0,0);
+        background: -moz-linear-gradient(90deg, rgba(0,0,0,1) 0%, rgba(0,0,0,0.6404762588629201) 35%, rgba(255,255,255,0) 100%);
+        background: -webkit-linear-gradient(90deg, rgba(0,0,0,1) 0%, rgba(0,0,0,0.6404762588629201) 35%, rgba(255,255,255,0) 100%);
+        background: linear-gradient(90deg, rgba(0,0,0,1) 0%, rgba(0,0,0,0.6404762588629201) 35%, rgba(255,255,255,0) 100%);
+        filter: progid:DXImageTransform.Microsoft.gradient(startColorstr="#000000",endColorstr="#ffffff",GradientType=1); 
+    }
+</style>
+
+<div class="d-flex flex-column body-p vh-100" loading="lazy">
+    <div class="d-flex flex-grow-1 div-p">
+        <form class="p-3 bg-body m-auto border border-2 border-info-subtle rounded bg-opacity-75" style="width: 24rem;" role="form" method="post" action="">
+            
+            <h3 class="fw-normal mb-3 pb-3" style="letter-spacing: 1px;">
+                <i class="nf nf-md-login"></i>
+                 Sign up
+            </h3>
+            
+            <div class="row mb-3">
+                <p class="mb-0 text-danger text-center">
+                    {% if msg %}
+                    {{ msg | safe }}
+                    {% else %}
+                    Enter your email and password to register
+                    {% endif %}
+                </p>
+            </div>
+
+            {% csrf_token %} 
+                            
+            <div class="input-group input-group-outline mb-3">
+                {{ form.username }}
+            </div>
+            <span class="text-danger">{{ form.username.errors }}</span>
+            
+            <div class="input-group input-group-outline mb-3">
+                {{ form.email }}
+            </div>
+            <span class="text-danger">{{ form.email.errors }}</span>
+            
+            <div class="input-group input-group-outline mb-3">
+                {{ form.password1 }}
+            </div>
+            <span class="text-danger">{{ form.password1.errors }}</span>
+            
+            <div class="input-group input-group-outline mb-3">
+                {{ form.password2 }}
+            </div>
+            <span class="text-danger">{{ form.password2.errors }}</span>
+            
+            <div class="input-group form-check form-check-info text-start">
+                <input class="form-check-input rounded" type="checkbox" value="" id="flexCheckDefault">
+                <label class="form-check-label ps-2" for="flexCheckDefault">
+                    I agree the <a href="javascript:;" class="text-info font-weight-bolder">Terms and Conditions</a>
+                </label>
+            </div>
+            <div class="text-center">
+                <button type="submit" name="register" 
+                class="btn btn-outline-primary bg-gradient-primary w-100 mt-4 mb-0 text-body-emphasis">Sign up</button>
+            </div>
+            <p class="mt-4 text-sm text-center">
+                Already have an account?
+                <a href="{% url 'login' %}" class="text-primary text-gradient font-weight-bold">Sign in</a>
+            </p>
+
+        </form>
+        
+    </div>
+    {% include 'includes/footer-auth.html' %} 
+    
+</div>
+
+{% endblock content %}
+
+<!-- Specific Page JS goes HERE  -->
+{% block javascripts %}{% endblock javascripts %}
+

archivist/apps/templates/files/upload.html

@@ -0,0 +1,59 @@
+{% extends "layouts/base-fullscreen.html" %}
+
+{% load static %}
+
+{% block title %} Upload {% endblock %} 
+
+<!-- Specific Page CSS goes HERE  -->
+{% block stylesheets %}{% endblock stylesheets %}
+
+{% block body_class %}{% endblock %}
+
+{% block content %}
+
+    {% include 'includes/navigation-transparent.html' %}
+
+    <style type="text/css">
+        .body-p{
+            background-image: url('{% static "/img/bg/login-bg.jpg" %}');
+            background-repeat: no-repeat;
+            background-attachment: fixed;
+            background-position: center; 
+            background-size: cover;
+        }
+
+        .div-p{
+            max-width:700px;
+            background: rgb(0,0,0);
+            background: -moz-linear-gradient(90deg, rgba(0,0,0,1) 0%, rgba(0,0,0,0.6404762588629201) 35%, rgba(255,255,255,0) 100%);
+            background: -webkit-linear-gradient(90deg, rgba(0,0,0,1) 0%, rgba(0,0,0,0.6404762588629201) 35%, rgba(255,255,255,0) 100%);
+            background: linear-gradient(90deg, rgba(0,0,0,1) 0%, rgba(0,0,0,0.6404762588629201) 35%, rgba(255,255,255,0) 100%);
+            filter: progid:DXImageTransform.Microsoft.gradient(startColorstr="#000000",endColorstr="#ffffff",GradientType=1); 
+        }
+    </style>
+    
+    <div class="d-flex flex-column body-p vh-100" loading="lazy">
+        <div class="d-flex flex-grow-1 div-p">
+            <form class="p-3 bg-body m-auto border border-2 border-info-subtle rounded bg-opacity-75" style="width: 24rem;" role="form" method="post" action="" enctype="multipart/form-data">
+                    {% if error_message %}
+                        <div class="alert alert-danger" role="alert">
+                            {{ error_message }}
+                        </div>
+                    {% endif %}
+                    {% csrf_token %}
+                    {{ form.as_p }}
+                    <button 
+                        type="submit"
+                        class="btn btn-primary bg-gradient-primary w-100 my-4 mb-2 text-light">
+                        Upload
+                    </button>
+            </form>
+            
+        </div>
+        {% comment %} {% include 'includes/footer.html' %} {% endcomment %} 
+    </div>
+
+{% endblock content %}
+
+<!-- Specific Page JS goes HERE  -->
+{% block javascripts %}{% endblock javascripts %}

archivist/apps/templates/importer/index.html

@@ -0,0 +1,55 @@
+{% extends "layouts/base-electric.html" %}
+
+{% load static %}
+
+{% block title %} Importer {% endblock title %}
+
+{% block stylesheets %}{% endblock stylesheets %}
+
+{% block content %}
+
+    {% include "includes/navigation-transparent.html" %}
+
+    <div class="container-fluid">
+        <div class="row row-gap-3">
+
+            <div class="col">
+                <div class="e-container-border e-container-radius">
+                    <div class="e-container e-container-radius p-2 pt-3 mb-3">
+                        <h1 class="text-center">Importer</h1>
+
+                        {% include "importer/partials/tabnavbar.html" %}
+
+                        <form class="p-3 m-auto border border-2 border-info-subtle rounded gap-2" style="width: 24rem;" role="form" method="post" action="">
+
+                            {% if ImportURLFormMSG %}
+                            <div class="row mb-3">
+                                <p class="mb-0 text-danger text-center">
+                                    {{ ImportURLFormMSG | safe }}
+                                </p>
+                            </div>
+                            {% endif %}
+
+                            {% csrf_token %}
+
+                            <div class="input-group mb-3" {% if form.url.errors %} style="border-color: red" {% endif %}>
+                                <label class="input-group-text" for="{{ form.url.id_for_label }}">{{ ImportURLForm.url.label }}</label>
+                                {{ ImportURLForm.url }}
+                            </div>
+                                {% comment %} <span class="input-group-text" id="basic-addon1">URL</span> {% endcomment %}
+                            <div class="d-grid">
+                                <button class="btn btn-primary" type="submit">Submit</button>
+                            </div>
+
+                        </form>
+
+                        <hr>
+
+                    </div>
+                </div>
+            </div>
+
+        </div>
+    </div>
+
+{% endblock content %}

archivist/apps/templates/importer/partials/tabnavbar.html

@@ -0,0 +1,21 @@
+<ul class="nav nav-tabs mb-3">
+{% for tab in tabs %}
+    <li class="nav-item">
+        {% if not tab.adminOnly %}
+            {% url tab.url as tab_url %}
+            {% if tab_url == request.path %}
+                <a class="nav-link active" aria-current="page" href="{% url tab.url %}">{{ tab.name }}</a>
+            {% else %}
+                <a class="nav-link" href="{% url tab.url %}">{{ tab.name }}</a>
+            {% endif %}
+        {% elif request.user.is_staff and request.user.is_superuser %}
+            {% url tab.url as tab_url %}
+            {% if tab_url == request.path %}
+                <a class="nav-link active" aria-current="page" href="{% url tab.url %}">{{ tab.name }}</a>
+            {% else %}
+                <a class="nav-link" href="{% url tab.url %}">{{ tab.name }}</a>
+            {% endif %}
+        {% endif %}
+    </li>
+{% endfor %}
+</ul>

archivist/apps/templates/importer/source_urls.html

@@ -0,0 +1,252 @@
+{% extends "layouts/base-electric.html" %}
+
+{% load static %}
+
+{% block title %} Source URLs | Importer {% endblock title %}
+
+{% block stylesheets %}{% endblock stylesheets %}
+
+{% block content %}
+
+{% include "includes/navigation-transparent.html" %}
+
+<div class="container-fluid">
+    <div class="row row-gap-3">
+
+        <div class="col">
+            <div class="e-container-border e-container-radius">
+                <div class="e-container e-container-radius p-2 pt-3 mb-3">
+                    <h1 class="text-center">Source URLs</h1>
+
+                    {% include "importer/partials/tabnavbar.html" %}
+
+                    <h3>Complete Profiles/Galleries</h3>
+
+                    <input type="text" class="form-control input-group-text my-2" onkeyup="filterTable('profiles', 0, this.value)" placeholder="Search for URLs...">
+
+                    <div class="table-responsive">
+                        <table id="profiles" class="table table-sm table-striped table-bordered table-responsive table-striped tabel-hover text-nowrap">
+                            <thead>
+                                <tr>
+                                    <th>URL:<span class="ms-1 text-primary">↕</span></th>
+                                    <th>Category:<span class="ms-1 text-primary">↕</span></th>
+                                    <th>Added On:<span class="ms-1 text-primary">↕</span></th>
+                                    <th data-bs-toggle="tooltip" title="Last Imported/Scaned">Imported:<span class="ms-1 text-primary">↕</span></th>
+                                    {% if user.is_superuser or user.is_staff %}
+                                    <th>Added By:<span class="ms-1 text-primary">↕</span></th>
+                                    {% endif %}
+                                    <th>Active:<span class="ms-1 text-primary">↕</span></th>
+                                </tr>
+                            </thead>
+                            <tbody>
+                                {% for url in SourceURLs %}
+                                {% if url.source_type == "C" %}
+                                <tr id="C-{{ url.pk }}">
+                                    <td>{{ url.url }}</td>
+                                    <td>{{ url.category }}</td>
+                                    <td data-timestamp="{{ url.date_added|date:'U' }}" data-bs-toggle="tooltip" title="{{ url.date_added|date:'Y-m-d H:i' }}">{{ url.date_added|date:'Y-m-d' }}</td>
+                                    <td data-timestamp="{{ url.last_imported|date:'U' }}" data-bs-toggle="tooltip" title="{{ url.last_imported|date:'Y-m-d H:i' }}">{{ url.last_imported|date:'Y-m-d' }}</td>
+                                    {% if user.is_superuser or user.is_staff %}
+                                    <td>{{ url.added_by_user|capfirst }}</td>
+                                    {% endif %}
+                                    <td>{{ url.active }}</td>
+                                </tr>
+                                {% endif %}
+                                {% endfor %}
+                            </tbody>
+                        </table>
+                    </div>
+
+                    <hr class="my-3">
+
+                    <h3>Single Posts</h3>
+
+                    <input type="text" class="form-control input-group-text my-2" onkeyup="filterTable('posts', 0, this.value)" placeholder="Search for URLs...">
+
+                    <table id="posts" class="table table-bordered table-responsive table-striped tabel-hover">
+                        <thead>
+                            <tr>
+                                <th>URL:</th>
+                                <th>Category:</th>
+                                <th>Added On:</th>
+                                <th>Last Imported/Scaned:</th>
+                                {% if user.is_superuser or user.is_staff %}
+                                <th>Added By</th>
+                                {% endif %}
+                                <th>Active</th>
+                            </tr>
+                        </thead>
+                        <tbody>
+                            {% for url in SourceURLs %}
+                            {% if url.source_type == "P" %}
+                            <tr>
+                                <td>{{ url.url }}</td>
+                                <td>{{ url.category }}</td>
+                                <td>{{ url.date_added|date:'Y-m-d H:i' }}</td>
+                                <td>{{ url.last_imported|date:'Y-m-d H:i' }}</td>
+                                {% if user.is_superuser or user.is_staff %}
+                                <td>{{ url.added_by_user|capfirst }}</td>
+                                {% endif %}
+                                <td>{{ url.active }}</td>
+                            </tr>
+                            {% endif %}
+                            {% endfor %}
+                        </tbody>
+                    </table>
+
+                </div>
+            </div>
+        </div>
+
+    </div>
+</div>
+
+<script>
+    function filterTable(tableId, columnIndex, input) {
+        console.log(tableId, columnIndex, input.value);
+        // Declare variables
+        var filter = input.toUpperCase();
+        var table = document.getElementById(tableId);
+        var rows = table.getElementsByTagName("tr");
+        
+        // Loop through all table rows, and hide those who don't match the search query
+        for (var i = 0; i < rows.length; i++) {
+            var cells = rows[i].getElementsByTagName("td");
+            if (cells.length) {
+                var txtValue = cells[columnIndex].textContent || cells[columnIndex].innerText;
+                if (txtValue.toUpperCase().indexOf(filter) > -1) {
+                    rows[i].style.display = "";
+                } else {
+                    rows[i].style.display = "none";
+                }
+            }
+        }
+    }
+
+    function sortTable(tableId, n) {
+        var table, rows, switching, i, x, y, shouldSwitch, dir, switchcount = 0;
+        table = document.getElementById(tableId);
+        switching = true;
+        // Set the sorting direction to ascending:
+        dir = "asc";
+        /* Make a loop that will continue until
+        no switching has been done: */
+        while (switching) {
+            // Start by saying: no switching is done:
+            switching = false;
+            rows = table.rows;
+            /* Loop through all table rows (except the
+            first, which contains table headers): */
+            for (i = 1; i < (rows.length - 1); i++) {
+                // Start by saying there should be no switching:
+                shouldSwitch = false;
+                /* Get the two elements you want to compare,
+                one from current row and one from the next: */
+                x = rows[i].getElementsByTagName("TD")[n];
+                y = rows[i + 1].getElementsByTagName("TD")[n];
+                /* Check if the two rows should switch place,
+                based on the direction, asc or desc: */
+                if (dir == "asc") {
+                    if (x.innerHTML.toLowerCase() > y.innerHTML.toLowerCase()) {
+                        // If so, mark as a switch and break the loop:
+                        shouldSwitch = true;
+                        break;
+                    }
+                } else if (dir == "desc") {
+                    if (x.innerHTML.toLowerCase() < y.innerHTML.toLowerCase()) {
+                        // If so, mark as a switch and break the loop:
+                        shouldSwitch = true;
+                        break;
+                    }
+                }
+            }
+            if (shouldSwitch) {
+                /* If a switch has been marked, make the switch
+                and mark that a switch has been done: */
+                rows[i].parentNode.insertBefore(rows[i + 1], rows[i]);
+                switching = true;
+                // Each time a switch is done, increase this count by 1:
+                switchcount ++;
+            } else {
+                /* If no switching has been done AND the direction is "asc",
+                set the direction to "desc" and run the while loop again. */
+                if (switchcount == 0 && dir == "asc") {
+                    dir = "desc";
+                    switching = true;
+                }
+            }
+        }
+    }
+
+    window.onload = function() {
+        document.querySelectorAll('th').forEach((element) => { // Table headers
+            element.addEventListener('click', function() {
+                let table = this.closest('table');
+
+                // If the column is sortable
+                if (this.querySelector('span')) {
+                    let order_icon = this.querySelector('span');
+                    let order      = encodeURI(order_icon.innerHTML).includes('%E2%86%91') ? 'desc' : 'asc';
+                    let separator  = '-----'; // Separate the value of it's index, so data keeps intact
+
+                    let value_list = {}; // <tr> Object
+                    let obj_key    = []; // Values of selected column
+
+                    let string_count = 0;
+                    let number_count = 0;
+
+                    // <tbody> rows
+                    table.querySelectorAll('tbody tr').forEach((line, index_line) => {
+                        // Value of each field
+                        let key = line.children[element.cellIndex].textContent.toUpperCase();
+
+                        // Check if value is date, numeric or string
+                        if (line.children[element.cellIndex].hasAttribute('data-timestamp')) {
+                            // if value is date, we store it's timestamp, so we can sort like a number
+                            key = line.children[element.cellIndex].getAttribute('data-timestamp');
+                        }
+                        else if (key.replace('-', '').match(/^[0-9,.]*$/g)) {
+                            number_count++;
+                        }
+                        else {
+                            string_count++;
+                        }
+
+                        value_list[key + separator + index_line] = line.outerHTML.replace(/(\t)|(\n)/g, ''); // Adding <tr> to object
+                        obj_key.push(key + separator + index_line);
+                    });
+                    if (string_count === 0) { // If all values are numeric
+                        console.log(obj_key);
+                        obj_key.sort(function(a, b) {
+                            return a.split(separator)[0] - b.split(separator)[0];
+                        });
+                        console.log(obj_key);
+                    }
+                    else {
+                        console.log(obj_key);
+                        obj_key.sort();
+                        console.log(obj_key);
+                    }
+
+                    if (order === 'desc') {
+                        console.log(obj_key);
+                        obj_key.reverse();
+                        console.log(obj_key);
+                        order_icon.innerHTML = '&darr;';
+                    }
+                    else {
+                        order_icon.innerHTML = '&uarr;';
+                    }
+
+                    let html = '';
+                    obj_key.forEach(function(chave) {
+                        html += value_list[chave];
+                    });
+                    table.getElementsByTagName('tbody')[0].innerHTML = html;
+                }
+            });
+        });
+    }
+</script>
+
+{% endblock content %}

archivist/apps/templates/includes/footer-auth.html

@@ -0,0 +1,22 @@
+<!-- Start footer auth -->
+<footer class="footer bg-body-tertiary bottom-2 py-2 w-100 z-index-3 border border-top-1">
+    <div class="container text-body-secondary">
+        <div class="row align-items-center justify-content-lg-between">
+            <div class="col-12 col-md-6 my-auto">
+                <div class="copyright text-center text-sm text-lg-start">
+                    &copy; 
+                    <a href="https://aroy-art.com" class="font-weight-bold" target="_blank">Aroy Art</a>
+                    and Contributors.
+                </div>
+            </div>
+            <div class="col-12 col-md-6">
+                <ul class="nav nav-footer justify-content-center justify-content-lg-end">
+                    <li class="nav-item">
+                        <a href="https://git.aroy-art.com/Aroy/Gallery-Archivist" class="nav-link" target="_blank">What is this? - Source</a>
+                    </li>
+                </ul>
+            </div>
+        </div>
+    </div>
+</footer>
+<!-- End footer auth -->

archivist/apps/templates/includes/gallery.html

@@ -0,0 +1,99 @@
+
+{% load static %}
+
+{% load media_filters %}
+
+{% load string_helper %}
+
+{% load thumbnail %}
+
+<div class="gallery-container">
+    {% for submission in submissions %}
+    <div class="gallery-item bg-dark">
+        
+        {% include "sites/partials/site-btn-overlay.html" with category=submission.category.name %}
+
+        <span class="seen-overlay text-primary-emphasis" data-seen="false" data-hash="{{ submission.submission_hash }}" href=''></span>
+
+        {% if submission.content_object.files.exists %}
+            
+            {% if submission.content_object.files.first.file_mime|is_image %}
+
+                {% if submission.content_object.files.all|length == 1 %}
+                    {% thumbnail submission.content_object.files.first.file "350" as im %}
+                        {% if submission.mature == True and user_profile.show_mature == "B" %}
+                            <img src="{{ im.url }}" alt="{{ submission.content_object.files.first.file_name }}" height="100%" class="blur">    
+                        {% else %}
+                            <img src="{{ im.url }}" alt="{{ submission.content_object.files.first.file_name }}" height="100%">
+                        {% endif %}
+                    {% endthumbnail %}
+
+                {% elif submission.content_object.files.all|length == 2 %}
+                    {% for file in submission.content_object.files.all %}
+                        {% thumbnail file.file "350" as im %}
+                            {% if submission.mature == True and user_profile.show_mature == "B" %}
+                                <img src="{{ im.url }}" alt="{{ file.file_name }}" height="100%" class="blur">    
+                            {% else %}
+                                <img src="{{ im.url }}" alt="{{ file.file_name }}" height="100%">
+                            {% endif %}
+                        {% endthumbnail %}
+                    {% endfor %}
+
+                {% else %}
+                    {% for file in submission.content_object.files.all %}
+                        {% thumbnail file.file "350" as im %}
+                            <div class="col">
+                                {% if submission.mature == True and user_profile.show_mature == "B" %}
+                                    <img src="{{ im.url }}" alt="{{ file.file_name }}" height="100%" class="blur">    
+                                {% else %}
+                                    <img src="{{ im.url }}" alt="{{ file.file_name }}" height="100%">
+                                {% endif %}
+                            </div>
+                        {% endthumbnail %}
+                    {% endfor %}
+
+                {% endif %}
+
+            {% elif submission.content_object.files.first.file_mime|is_video %}
+
+                <video class="gallery-item" src="{% url 'files:serve_content_file' 'submission' submission.content_object.files.first.file_hash %}" controlsList="nodownload"></video>
+            
+            {% else %}
+                <img src="{% static 'img/placeholder/no-image-dark.webp' %}">
+            
+            {% endif %}
+
+        {% else %}
+        
+            <span class="badge bg-secondary">This submission has no media</span>
+        
+        {% endif %}
+        <a href='{% url "sites:submission" submission.submission_hash %}' class="stretched-link"></a>
+        <div class="overlay p-2 text-center">
+            {% if submission.content_object.title %}
+                {% if submission.content_object.title|length > 64 %}
+                    <p title="{{ submission.content_object.title }}">{{ submission.content_object.title|slice:"0:64"|add:"..." }}</p>
+                {% else %}
+                    <p title="{{ submission.content_object.title }}">{{ submission.content_object.title }}</p>
+                {% endif %}
+
+            {% else %}
+                {% if submission.content_object.description|length > 64 %}
+                    <p>{{ submission.content_object.description|html_to_text|slice:"0:64"|add:"..." }}</p>
+                {% else %}
+                    <p>{{ submission.content_object.description|html_to_text }}</p>
+                {% endif %}
+
+            {% endif %}
+            <a href="{% url 'sites:artist_profile' submission.author.user_hash %}" class="z-2">
+                {% if submission.category.name == "furaffinity" %}
+                {{ submission.content_object.artist.artist }}
+                {% else %}
+                {{ submission.content_object.author.artist }}
+                {% endif %}
+            </a>
+            <small class="badge bg-secondary">{{ submission.content_object.date|date:'Y-m-d H:i:s' }}</small>
+        </div>
+    </div>
+    {% endfor %}
+</div>

archivist/apps/templates/includes/navigation-transparent.html

@@ -0,0 +1,196 @@
+<!-- Navbar Transparent -->
+<nav class="navbar navbar-expand-lg fixed-top top-0 z-index-3 w-100 shadow-none my-3 text-body">
+    <div class="container bg-body-secondary p-2 rounded bg-opacity-75 shadow">
+
+        <!-- svg icons -->
+        <svg xmlns="http://www.w3.org/2000/svg" class="d-none">
+            <symbol id="bootstrap" viewBox="0 0 512 408" fill="currentcolor">
+                <path d="M106.342 0c-29.214 0-50.827 25.58-49.86 53.32.927 26.647-.278 61.165-8.966 89.31C38.802 170.862 24.07 188.707 0 191v26c24.069 2.293 38.802 20.138 47.516 48.37 8.688 28.145 9.893 62.663 8.965 89.311C55.515 382.42 77.128 408 106.342 408h299.353c29.214 0 50.827-25.58 49.861-53.319-.928-26.648.277-61.166 8.964-89.311 8.715-28.232 23.411-46.077 47.48-48.37v-26c-24.069-2.293-38.765-20.138-47.48-48.37-8.687-28.145-9.892-62.663-8.964-89.31C456.522 25.58 434.909 0 405.695 0H106.342zm236.559 251.102c0 38.197-28.501 61.355-75.798 61.355h-87.202a2 2 0 01-2-2v-213a2 2 0 012-2h86.74c39.439 0 65.322 21.354 65.322 54.138 0 23.008-17.409 43.61-39.594 47.219v1.203c30.196 3.309 50.532 24.212 50.532 53.085zm-84.58-128.125h-45.91v64.814h38.669c29.888 0 46.373-12.03 46.373-33.535 0-20.151-14.174-31.279-39.132-31.279zm-45.91 90.53v71.431h47.605c31.12 0 47.605-12.482 47.605-35.941 0-23.46-16.947-35.49-49.608-35.49h-45.602z"/>
+            </symbol>
+            <symbol id="check2" viewBox="0 0 16 16">
+                <path d="M13.854 3.646a.5.5 0 0 1 0 .708l-7 7a.5.5 0 0 1-.708 0l-3.5-3.5a.5.5 0 1 1 .708-.708L6.5 10.293l6.646-6.647a.5.5 0 0 1 .708 0z"/>
+            </symbol>
+            <symbol id="circle-half" viewBox="0 0 16 16">
+                <path d="M8 15A7 7 0 1 0 8 1v14zm0 1A8 8 0 1 1 8 0a8 8 0 0 1 0 16z"/>
+            </symbol>
+            <symbol id="moon-stars-fill" viewBox="0 0 16 16">
+                <path d="M6 .278a.768.768 0 0 1 .08.858 7.208 7.208 0 0 0-.878 3.46c0 4.021 3.278 7.277 7.318 7.277.527 0 1.04-.055 1.533-.16a.787.787 0 0 1 .81.316.733.733 0 0 1-.031.893A8.349 8.349 0 0 1 8.344 16C3.734 16 0 12.286 0 7.71 0 4.266 2.114 1.312 5.124.06A.752.752 0 0 1 6 .278z"/>
+                <path d="M10.794 3.148a.217.217 0 0 1 .412 0l.387 1.162c.173.518.579.924 1.097 1.097l1.162.387a.217.217 0 0 1 0 .412l-1.162.387a1.734 1.734 0 0 0-1.097 1.097l-.387 1.162a.217.217 0 0 1-.412 0l-.387-1.162A1.734 1.734 0 0 0 9.31 6.593l-1.162-.387a.217.217 0 0 1 0-.412l1.162-.387a1.734 1.734 0 0 0 1.097-1.097l.387-1.162zM13.863.099a.145.145 0 0 1 .274 0l.258.774c.115.346.386.617.732.732l.774.258a.145.145 0 0 1 0 .274l-.774.258a1.156 1.156 0 0 0-.732.732l-.258.774a.145.145 0 0 1-.274 0l-.258-.774a1.156 1.156 0 0 0-.732-.732l-.774-.258a.145.145 0 0 1 0-.274l.774-.258c.346-.115.617-.386.732-.732L13.863.1z"/>
+            </symbol>
+            <symbol id="sun-fill" viewBox="0 0 16 16">
+                <path d="M8 12a4 4 0 1 0 0-8 4 4 0 0 0 0 8zM8 0a.5.5 0 0 1 .5.5v2a.5.5 0 0 1-1 0v-2A.5.5 0 0 1 8 0zm0 13a.5.5 0 0 1 .5.5v2a.5.5 0 0 1-1 0v-2A.5.5 0 0 1 8 13zm8-5a.5.5 0 0 1-.5.5h-2a.5.5 0 0 1 0-1h2a.5.5 0 0 1 .5.5zM3 8a.5.5 0 0 1-.5.5h-2a.5.5 0 0 1 0-1h2A.5.5 0 0 1 3 8zm10.657-5.657a.5.5 0 0 1 0 .707l-1.414 1.415a.5.5 0 1 1-.707-.708l1.414-1.414a.5.5 0 0 1 .707 0zm-9.193 9.193a.5.5 0 0 1 0 .707L3.05 13.657a.5.5 0 0 1-.707-.707l1.414-1.414a.5.5 0 0 1 .707 0zm9.193 2.121a.5.5 0 0 1-.707 0l-1.414-1.414a.5.5 0 0 1 .707-.707l1.414 1.414a.5.5 0 0 1 0 .707zM4.464 4.465a.5.5 0 0 1-.707 0L2.343 3.05a.5.5 0 1 1 .707-.707l1.414 1.414a.5.5 0 0 1 0 .708z"/>
+            </symbol>
+        </svg>
+        <!-- svg icons end -->
+
+        <a class="navbar-brand ms-2" href="{% url 'home' %}" rel="tooltip" title="Designed and Coded by Aroy" data-placement="bottom">
+            Gallery Archivists
+        </a>
+        <button class="navbar-toggler shadow-none ms-2" type="button" data-bs-toggle="collapse" data-bs-target="#navigation" aria-controls="navigation" aria-expanded="false" aria-label="Toggle navigation">
+            <span class="navbar-toggler-icon my-1">
+                <span class="navbar-toggler-bar bar1"></span>
+                <span class="navbar-toggler-bar bar2"></span>
+                <span class="navbar-toggler-bar bar3"></span>
+            </span>
+        </button>
+        <div class="collapse navbar-collapse w-100 pt-3 pb-2 py-lg-0 ms-lg-12 ps-lg-5" id="navigation">
+            <ul class="navbar-nav navbar-nav-hover ms-auto">
+                
+                {% if request.user.is_authenticated %}
+                
+                <!-- Search form -->
+                <li class="nav-item ms-lg-auto mx-2">
+                    <form class="d-flex" role="search" method="get" action="{% url 'sites:browse' %}">
+                        <div class="input-group me-2">
+                            <input class="form-control" type="search" name="q" placeholder="Search" aria-label="Search">
+                            <button class="input-group-text nf nf-fa-search" id="search-addon" type="submit"></button>
+                        </div>
+                    </form>
+                </li>
+            
+                <li class="nav-item py-2 py-lg-1 col-12 col-lg-auto">
+                    <div class="vr d-none d-lg-flex h-100 mx-lg-2 text-white"></div>
+                    <hr class="d-lg-none my-2">
+                </li>
+
+                <!-- Theme Selector -->
+                <li class="nav-item dropdown ms-auto mx-auto">    
+                    <div class="d-flex align-items-center dropdown-center ">
+                        <button class="btn btn-link text-body-emphasis px-0 text-decoration-none dropdown-toggle d-flex align-items-center icon-link"
+                                id="bd-theme"
+                                type="button"
+                                aria-expanded="false"
+                                data-bs-toggle="dropdown"
+                                data-bs-display="static"
+                                aria-label="Toggle theme">
+                            <svg class="bi my-1 theme-icon-active"><use href="#circle-half"></use></svg>
+                            <span class="d-lg-none ms-0" id="bd-theme-text"></span>
+                        </button>
+                        <ul class="dropdown-menu" aria-labelledby="bd-theme" >
+                            <li>
+                                <button type="button" class="dropdown-item d-flex align-items-center icon-link" data-bs-theme-value="light">
+                                    <svg class="bi me-2 opacity-50 theme-icon"><use href="#sun-fill"></use></svg>
+                                    Light
+                                    <svg class="bi ms-auto d-none"><use href="#check2"></use></svg>
+                                </button>
+                            </li>
+                            <li>
+                                <button type="button" class="dropdown-item d-flex align-items-center icon-link" data-bs-theme-value="dark">
+                                    <svg class="bi me-2 opacity-50 theme-icon"><use href="#moon-stars-fill"></use></svg>
+                                    Dark
+                                    <svg class="bi ms-auto d-none"><use href="#check2"></use></svg>
+
+                                </button>
+                            </li>
+                            <li>
+                                <button type="button" class="dropdown-item d-flex align-items-center active icon-link" data-bs-theme-value="auto">
+                                    <svg class="bi me-2 opacity-50 theme-icon"><use href="#circle-half"></use></svg>
+                                    Auto
+                                    <svg class="bi ms-auto d-none"><use href="#check2"></use></svg>
+                                </button>
+                            </li>
+                        </ul>
+                    </div>
+                </li>
+                <!-- End Theme Selector -->
+            
+                <li class="nav-item py-2 py-lg-1 col-12 col-lg-auto">
+                    <div class="vr d-none d-lg-flex h-100 mx-lg-2 text-white"></div>
+                    <hr class="d-lg-none my-2">
+                </li>
+            
+                <!-- User Dropdown -->
+                <li class="nav-item ms-lg-auto mx-2 dropdown">
+                    <a class="nav-link dropdown-toggle text-body-emphasis" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">
+                        {% if user.first_name %}
+                            {{ user.first_name }}
+                        {% else %}
+                            {{ user.username }}
+                        {% endif %}
+                    </a>
+
+                    <ul class="dropdown-menu dropdown-menu-end">
+
+                        <li>
+                            <a class="dropdown-item" href="{% url 'profile' %}">
+                                <i class="nf nf-fa-user"></i>
+                                <p class="d-inline font-weight-bold" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Settings">User Settings</p>
+                            </a>
+                        </li>
+
+                        <li>
+                            <hr class="dropdown-divider">
+                        </li>
+
+                        {% if user.is_superuser %}
+                        <li>
+                            <a class="dropdown-item" href="{% url 'admin:index' %}">
+                                <i class="nf nf-fa-warning"></i>
+                                <p class="d-inline font-weight-bold" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Django Admin">Djnago Admin</p>
+                            </a>
+                        </li>
+                        {% endif %}
+
+                        <li>
+                            <hr class="dropdown-divider">
+                        </li>
+
+                        <li>
+                            <a class="dropdown-item" href="{% url 'logout' %}">
+                                <i class="nf nf-md-logout"></i>
+                                <p class="d-inline font-weight-bold" data-bs-toggle="tooltip" data-bs-placement="bottom" title="Sign out">Logout</p>
+                            </a>
+                        </li>          
+                    </ul>
+                </li>
+                <!-- End User Dropdown -->
+            
+                {% else %}
+            
+                <!-- Theme Selector -->
+                <li class="nav-item ms-lg-auto mx-2">    
+                    <div class="d-flex align-items-center dropdown color-modes">
+                        <button class="btn btn-link text-body px-0 me-2 text-decoration-none dropdown-toggle d-flex align-items-center icon-link"
+                            id="bd-theme"
+                            type="button"
+                            aria-expanded="false"
+                            data-bs-toggle="dropdown"
+                            data-bs-display="static"
+                            aria-label="Toggle theme">
+                            <svg class="bi my-1 theme-icon-active"><use href="#circle-half"></use></svg>
+                            <span class="ms-2" id="bd-theme-text">Theme</span>
+                        </button>
+                        <ul class="dropdown-menu dropdown-menu-end" aria-labelledby="bd-theme">
+                            <li>
+                                <button type="button" class="dropdown-item d-flex align-items-center icon-link" data-bs-theme-value="light">
+                                    <svg class="bi me-2 opacity-50 theme-icon"><use href="#sun-fill"></use></svg>
+                                    Light
+                                    <svg class="bi ms-auto d-none"><use href="#check2"></use></svg>
+                                </button>
+                            </li>
+                            <li>
+                                <button type="button" class="dropdown-item d-flex align-items-center icon-link" data-bs-theme-value="dark">
+                                    <svg class="bi me-2 opacity-50 theme-icon"><use href="#moon-stars-fill"></use></svg>
+                                    Dark
+                                    <svg class="bi ms-auto d-none"><use href="#check2"></use></svg>    
+                                </button>
+                            </li>
+                            <li>
+                                <button type="button" class="dropdown-item d-flex align-items-center active icon-link" data-bs-theme-value="auto">
+                                    <svg class="bi me-2 opacity-50 theme-icon"><use href="#circle-half"></use></svg>
+                                    Auto
+                                    <svg class="bi ms-auto d-none"><use href="#check2"></use></svg>
+                                </button>
+                            </li>
+                        </ul>
+                    </div>
+                </li>
+                <!-- End Theme Selector -->
+
+                {% endif %}
+
+            </ul>
+        </div>
+    </div>
+</nav>
+<!-- End Navbar -->

archivist/apps/templates/includes/pageination.html

@@ -0,0 +1,90 @@
+
+<!-- Pagination -->
+<div class="row justify-content-center">
+
+    <div class="col">
+        <nav aria-label="Page navigation">
+            <ul class="pagination justify-content-center">
+                
+                {% if page_obj.has_previous %}
+                    <li class="page-item">
+                        <a class="page-link" href="?{{ request.GET.urlencode }}&page=1" aria-label="First">
+                            «
+                        </a>
+                    </li>
+                    <li class="page-item">
+                        <a class="page-link" href="?{{ request.GET.urlencode }}&page={{ page_obj.previous_page_number }}" aria-label="Previous">
+                            <span aria-hidden="true">
+                                ‹
+                            </span>
+                        </a>
+                    </li>
+
+                {% else %}
+                    <li class="page-item disabled">
+                        <span class="page-link text-decoration-line-through">
+                            «
+                        </span>
+                    </li>
+                    <li class="page-item disabled">
+                        <span class="page-link text-decoration-line-through">
+                            ‹
+                        </span>
+                    </li>
+                {% endif %}
+                
+                <li class="page-item disabled">
+                    <span class="page-link text-nowrap">
+                        Page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }}.
+                    </span>
+                    </li>
+            
+                {% if page_obj.has_next %}
+                    <li class="page-item">
+                        <a class="page-link" href="?{{ request.GET.urlencode }}&page={{ page_obj.next_page_number }}" aria-label="Next">
+                            <span aria-hidden="true">
+                                ›
+                            </span>
+                        </a>
+                    </li>
+                    <li class="page-item">
+                        <a class="page-link" href="?{{ request.GET.urlencode }}&page={{ page_obj.paginator.num_pages }}" aria-label="Last">
+                            »
+                        </a>
+                    </li>
+
+                {% else %}
+                    <li class="page-item disabled">
+                        <span class="page-link text-decoration-line-through">
+                            ›
+                        </span>
+                    </li>
+                    <li class="page-item disabled">
+                        <span class="page-link text-decoration-line-through">
+                            »
+                        </span>
+                    </li>
+                
+                {% endif %}
+                
+            </ul>
+        </nav>
+    </div>
+
+    <!-- Jump-to field -->
+    <div class="col ">
+        <form method="get" class="d-flex flex-row flex-nowrap flex-grow-1 justify-content-center ">
+            {% for key, value in request.GET.items %}
+                {% if key != "page" %}
+                    <input type="hidden" name="{{ key }}" value="{{ value }}">
+                {% endif %}
+            {% endfor %}
+            <label for="jumpToPage" class="col-auto col-form-label mx-1">Jump to:</label>
+            <input type="number" name="page" id="jumpToPage" class="form-control mx-1" min="1" max="{{ page_obj.paginator.num_pages }}" value="{{ page_obj.number }}" style="width: 5rem">
+            <button type="submit" class="btn btn-primary mx-1">Go</button>
+        </form>
+    </div>
+    <!-- Jump to field End -->
+
+</div>
+<!-- Pagination End -->

archivist/apps/templates/includes/scripts.html

@@ -0,0 +1,14 @@
+
+{% load static %}
+
+<!--   Core JS Files   -->
+
+<script src="{% static 'libs/bootstrap/bootstrap.bundle.min.js' %}" type="text/javascript"></script>
+
+<!-- Htmx JS & Extentions -->
+<script src="{% static 'libs/htmx/htmx.min.js' %}" type="text/javascript"></script>
+
+{% if DEBUG %}
+<!-- Htmx Debug JS -->
+<script src="{% static 'libs/htmx/debug.js' %}" type="text/javascript"></script>
+{% endif %}

archivist/apps/templates/layouts/base-electric.html

@@ -0,0 +1,74 @@
+{% load static %}
+
+<!DOCTYPE html>
+<html lang="en" itemscope itemtype="http://schema.org/WebPage" data-bs-theme="auto">
+<head>
+	<meta charset="utf-8"/>
+    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"/>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+
+	<meta name="description" content="{% block meta_description %}{% endblock meta_description %}">
+	<meta name="keywords" content="{% block meta_keywords %}{% endblock meta_keywords %}">
+	
+	<link rel="apple-touch-icon" sizes="76x76" href="{{ ASSETS_ROOT }}/img/apple-icon.png"/>
+    <link rel="icon" type="image/png" href='{% static "/img/favicon.png" %}'/>
+    
+    <title>
+        {% block title %}{% endblock title %} - Gallery-Archivists
+    </title>
+
+	<!-- Fonts and icons -->
+    <link rel="stylesheet" type="text/css" href="https://fonts.googleapis.com/css?family=Roboto:300,400,500,700,900|Roboto+Slab:400,700" />
+    
+    <!-- Nerd Fonts-->
+    <link rel="stylesheet" href="{% static 'libs/nerdfonts/nerd-fonts-generated.min.css' %}"/>
+    
+    <!-- CSS Files -->
+    <link rel="stylesheet" href="{% static 'libs/bootstrap/bootstrap.min.css' %}"/>
+
+	<!-- Main CSS File  -->
+	<link rel="stylesheet" href="{% static 'css/main.css' %}"/>
+
+	<!-- Specific Page CSS goes HERE  -->
+    {% block stylesheets %}{% endblock stylesheets %}
+
+</head>
+<body class="{% block body_class %}{% endblock body_class %}" hx-header='{% block hx_header %}{% endblock hx_header %} {"X-CSRFToken": "{{ csrf_token }}"}'>
+	<div class="everything">
+		<div class="wires">
+			<!-- Content -->
+    		{% block content %}{% endblock content %}  
+    		<!-- End Content -->
+		</div>
+    </div>
+
+	<div class="modal fade" id="externalLinkConfirmationModal" data-bs-backdrop="static" data-bs-keyboard="false" tabindex="-1" role="dialog" aria-labelledby="externalLinkConfirmationModalLabel" aria-hidden="true">
+        <div class="modal-dialog modal-dialog-centered modal-dialog-scrollable" role="document">
+            <div class="modal-content">
+                <div class="modal-header bg-danger">
+                    <h5 class="modal-title" id="externalLinkConfirmationModalLabel">Confirmation</h5>
+                    <button type="button" class="btn-close btn-close-white" data-bs-dismiss="modal" aria-label="Close"></button>
+                </div>
+                <div class="modal-body text-body-emphasis">
+                    <p>You are leaving this site and visiting an external link. Do you want to proceed?</p>
+                    <p id="externalLinkShow"></p>
+                </div>
+                <div class="modal-footer bg-warning">
+                    <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
+                    <a id="externalLink" href="#" target="_blank" rel="noopener noreferrer" class="btn btn-primary">Proceed</a>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    {% include "includes/scripts.html" %}
+
+    <script src='{% static "js/confirm_external_links.js" %}'></script>
+    <script src='{% static "js/color-modes.js" %}'></script>
+    <script src='{% static "js/main.js" %}'></script>
+
+	<!-- Specific Page JS goes HERE  -->
+	{% block scripts %}{% endblock scripts %}
+
+</body>
+</html>

archivist/apps/templates/layouts/base-fullscreen.html

@@ -0,0 +1,65 @@
+{% load static %}
+
+<!DOCTYPE html>
+<html lang="en" itemscope itemtype="http://schema.org/WebPage" data-bs-theme="auto">
+
+<head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+    <link rel="apple-touch-icon" sizes="76x76" href="{{ ASSETS_ROOT }}/img/apple-icon.png">
+    <link rel="icon" type="image/png" href="{{ ASSETS_ROOT }}/img/favicon.png">
+    
+    <title>
+        {% block title %}{% endblock title %} - Gallery-Archivists
+    </title>
+    
+    <!-- Fonts and icons -->
+    <link rel="stylesheet" type="text/css" href="https://fonts.googleapis.com/css?family=Roboto:300,400,500,700,900|Roboto+Slab:400,700" />
+    
+    <!-- Nerd Fonts-->
+    <link rel="stylesheet" href="{% static 'libs/nerdfonts/nerd-fonts-generated.min.css' %}">
+    
+    <!-- CSS Files -->
+    <link rel="stylesheet" href="{% static 'libs/bootstrap/bootstrap.min.css' %}">
+    
+    <!-- Specific Page CSS goes HERE  -->
+    {% block stylesheets %}{% endblock stylesheets %}
+    
+</head>
+
+<body class="{% block body_class %} {% endblock body_class %} ">
+    <!-- Content -->
+    {% block content %}{% endblock content %}  
+    <!-- End Content -->
+    
+    
+    <div class="modal fade" id="externalLinkConfirmationModal" data-bs-backdrop="static" data-bs-keyboard="false" tabindex="-1" role="dialog" aria-labelledby="externalLinkConfirmationModalLabel" aria-hidden="true">
+        <div class="modal-dialog modal-dialog-centered modal-dialog-scrollable" role="document">
+            <div class="modal-content">
+                <div class="modal-header bg-danger">
+                    <h5 class="modal-title" id="externalLinkConfirmationModalLabel">Confirmation</h5>
+                    <button type="button" class="btn-close btn-close-white" data-bs-dismiss="modal" aria-label="Close"></button>
+                </div>
+                <div class="modal-body">
+                    <p>You are leaving this site and visiting an external link. Do you want to proceed?</p>
+                    <p id="externalLinkShow"></p>
+                </div>
+                <div class="modal-footer bg-warning">
+                    <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
+                    <a id="externalLink" href="#" target="_blank" rel="noopener noreferrer" class="btn btn-primary">Proceed</a>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <script src='{% static "js/confirm_external_links.js" %}'></script>
+    <script src='{% static "js/color-modes.js" %}'></script>
+    <script src='{% static "js/main.js" %}'></script>
+
+    {% include "includes/scripts.html" %} 
+
+    <!-- Specific Page JS goes HERE  -->
+    {% block javascripts %}{% endblock javascripts %}
+    
+</body>
+</html>

archivist/apps/templates/sites/browse.html

@@ -0,0 +1,90 @@
+{% extends "layouts/base-electric.html" %}
+
+{% load static %}
+
+{% block title %}Browse{% endblock title %}
+
+{% block stylesheets %}{% endblock stylesheets %}
+
+{% block content %}
+
+    {% include "includes/navigation-transparent.html" %}
+
+    <div class="container-fluid"> 
+        <div class="row row-gap-3 column-gap-0">
+            
+            <div class="col-xl-9 col-lg-8 pe-lg-0">
+                <div class="e-container-border e-container-radius">
+                    <div class="e-container e-container-radius p-2 pt-3 mb-3">
+                        <h1 class="text-center">Browse</h1>
+
+                        <hr>
+
+                        {% include "includes/pageination.html" with page_obj=submissions %}
+
+                        {% include "includes/gallery.html" with user_profile=user_profile %}
+
+                        {% include "includes/pageination.html" with page_obj=submissions %}
+
+                    </div>
+                </div>
+            </div>
+            
+            <div class="col-xl-3 col-lg-4">
+                <div class="e-container-border e-container-radius d-none d-sm-none d-md-none d-lg-block ">
+                    <div class="e-container e-container-radius p-2 pt-3 mb-3 ">
+                        <h1 class="text-center">Search</h1>
+
+                        <hr>
+
+                        <form class="d-flex flex-column gap-2 bg-body-secondary p-2 rounded" role="search" method="get" action="{% url 'sites:browse' %}">
+
+                            <div class="input-group">
+                                {{ form.q }}
+                                <button class="input-group-text nf nf-fa-search" id="search-addon" type="submit"></button>
+                            </div>
+                            
+                            <div class="input-group">
+                                <label class="input-group-text" for="type">{{ form.sort.label }}:</label>
+                                <div class="form-control pt-2">
+                                    {% for radio in form.sort %}
+                                    <div class="form-check form-check-inline">
+                                        <label for="{{ radio.id_for_label }}">{{ radio.choice_label }}</label>
+                                        <input class="form-check-input" type="radio" name="sort" value="{{ radio.data.value }}" id="{{ radio.id_for_label }}" {% if radio.data.selected %}checked{% endif %}>
+                                    </div>
+                                    {% endfor %}
+                                </div>
+                            </div>
+
+                            <div class="input-group">
+                                <label class="input-group-text" for="category">{{ form.category.label }}:</label>
+                                {{ form.category }}
+                            </div>
+
+                            <div class="input-group">
+                                <label class="input-group-text" for="mature">{{ form.mature.label }}:</label>
+                                
+                                <div class="form-control pt-2">
+                                    
+                                {% for radio in form.mature %}
+                                    <div class="form-check form-check-inline">
+                                        <label for="{{ radio.id_for_label }}">{{ radio.choice_label }}</label>
+                                        <input class="form-check-input" type="radio" name="mature" value="{{ radio.data.value }}" id="{{ radio.id_for_label }}" {% if radio.data.selected %}checked{% endif %}>
+                                    </div>
+                                {% endfor %}
+                                
+                                </div>
+                            </div>
+
+                        </form>
+
+                        <hr>
+
+                    </div>
+                </div>
+            </div>
+
+        </div>
+    </div>
+
+{% endblock content %}

archivist/apps/templates/sites/partials/post-info.html

@@ -0,0 +1,117 @@
+{% load string_helper %}
+
+<h1 class="text-center m-0">Post Info:</h1>
+
+<hr class="m-1">
+
+<div class="m-2 overflow-hidden">
+    <strong class="text-info">Tags: </strong>
+    {% if submission.content_object.tags.exists %}
+        {% for tag in submission.content_object.tags.all %}
+            <span>
+                <a class="badge bg-primary tag" href="{% url 'sites:tag' tag.tag_slug %}">{{ tag.tag|capfirst }}</a>
+            </span>
+        {% endfor %}
+    {% else %}
+        <span class="badge bg-secondary">This submission has no tags</span>
+    {% endif %}
+</div>
+
+<hr class="m-2">
+
+<table class="table table-sm rounded-2 bg-secondary">
+    <tbody class="rounded-2">
+        <tr>
+            {% if submission.category.name == "furaffinity" %}
+            <th scope="row" class="text-info">FurAffinity ID</th>
+            {% elif submission.category.name == "twitter" %}
+            <th scope="row" class="text-info">Twitter ID</th>
+            {% elif submission.category.name == "instagram" %}
+            <th scope="row" class="text-info">Instagram ID</th>
+            {% else %}
+            <th scope="row" class="text-info">Submission ID</th>
+            {% endif %}
+            <td>{{ submission.content_object.submission_id }}</td>
+        </tr>
+        <tr>
+            {% if submission.category.name == "furaffinity" %}
+            <th scope="row" class="text-info">Views</th>
+            <td>{{ submission.content_object.views }}</td>
+            {% endif %}
+        </tr>
+        <tr>
+            <th scope="row" class="text-info">Gallery Type</th>
+            <td>{{ submission.content_object.gallery_type|capfirst }}</td>
+        </tr>
+        <tr>
+            <th scope="row" class="text-info">Lang</th>
+            <td>{{ submission.content_object.lang }}</td>
+        </tr>
+        <tr>
+            <th scope="row" class="text-info">Favorites</th>
+            <td>{{ submission.content_object.favorites_count }}</td>
+        </tr>
+        <tr>
+            <th scope="row" class="text-info">Retweets</th>
+            <td>{{ submission.content_object.retweet_count }}</td>
+        </tr>
+        <tr>
+            <th scope="row" class="text-info">Quotes</th>
+            <td>{{ submission.content_object.quote_count }}</td>
+        </tr>
+        <tr>
+            <th scope="row" class="text-info">Replies</th>
+            <td>{{ submission.content_object.reply_count }}</td>
+        </tr>
+    </tbody>
+</table>
+
+<h1 class="text-center m-0">Media Info:</h1>
+
+<hr class="m-1">
+
+<table class="table table-sm">
+    <tbody>
+        {% if submission.content_object.files.all|length == 0 %}
+            <tr>
+                <th scope="row" class="text-warning">No Media</th>
+            </tr>
+        {% elif submission.content_object.files.all|length <= 1 %}
+            <tr>
+                <th scope="row" class="text-info">Image Size</th>
+                <td>{{ submission.content_object.files.first.image_width }} x {{ submission.content_object.files.first.image_height }}</td>
+            </tr>
+            <tr>
+                <th scope="row" class="text-info">Size</th>
+                <td>{{ submission.content_object.files.first.size|size_to_human_readable }}</td>
+            </tr>
+            
+
+        {% else %}
+            {% for file in submission.content_object.files.all %}
+                <tr>
+                    <th scope="row" class="text-info">Image Res {{ forloop.counter }}</th>
+                    <td>{{ file.image_width }} x {{ file.image_height }}</td>
+                </tr>
+                <tr>
+                    <th scope="row" class="text-info">Size {{ forloop.counter }}</th>
+                    <td>{{ file.size|size_to_human_readable }}</td>
+                </tr>
+                
+            {% endfor %}
+
+        {% endif %}
+        <tr>
+            <th scope="row" class="text-info">Mature</th>
+            <td><span class="badge {% if submission.content_object.sensitive %}bg-danger{% else %}bg-success{% endif %} text-2xl">{{ submission.content_object.sensitive|default_if_none:False }}</span></td>
+        </tr>
+        <tr>
+            <th scope="row" class="text-info">Orginal Date</th>
+            <td>{{ submission.content_object.date |date:'Y-m-d H:i:s' }}</td>
+        </tr>
+        <tr>
+            <th scope="row" class="text-info">Archive Date</th>
+            <td>{{ submission.date_added |date:'Y-m-d H:i:s' }}</td>
+        </tr>
+    </tbody>
+</table>

archivist/apps/templates/sites/partials/site-btn-overlay.html

@@ -0,0 +1,15 @@
+
+{% load static %}
+
+<a class="site-btn-overlay" href='{% url "sites:site_overview" category %}'>
+    {% if category == "twitter" %} 
+        <img src='{% static "/img/site-logos/twitter_logo.png" %}' alt="{{ category }}"/>
+
+    {% elif category == "furaffinity" %}
+        <img src='{% static "/img/site-logos/fa_logo.png" %}' alt="{{ category }}"/>
+
+    {% else %}
+        <small>{{ category|capfirst }}</small>
+    
+    {% endif %}
+</a>

archivist/apps/templates/sites/tags_list.html

@@ -0,0 +1,32 @@
+{% extends "layouts/base-electric.html" %}
+
+{% load static %}
+
+{% load string_helper %}
+
+{% block title %}
+    Tags
+{% endblock title %}
+
+
+{% block content %}
+
+    {% include "includes/navigation-transparent.html" %}
+    
+    <div class="container-fluid"> 
+        <div class="e-container-border e-container-radius row m-0 mb-3" tabindex="1">
+            <div class="e-container e-container-radius p-4 overflow-x-hidden">
+                <h1 class="text-center">Tags</h1>
+                
+                <hr>
+
+                <div class="mt-2 text-center">
+                    {% for tag in tags %}
+                         <a class="badge bg-primary text-decoration-none" href="{% url 'sites:tag' tag.tag_slug %}">{{ tag|capfirst }}</a>
+                    {% endfor %}
+                </div>
+            </div>
+        </div>
+    </div>
+
+{% endblock content %}

archivist/apps/user/admin.py

@@ -0,0 +1,15 @@
+from django.contrib import admin
+
+from .models import UserProfile, SeenPost
+
+# Register your models here.
+
+class UserProfileAdmin(admin.ModelAdmin):
+    list_display = ('user', 'show_mature')
+    
+class SeenPostAdmin(admin.ModelAdmin):
+    list_display = ('user', 'post', 'timestamp')
+
+
+admin.site.register(UserProfile, UserProfileAdmin)
+admin.site.register(SeenPost, SeenPostAdmin)

archivist/apps/user/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class UsersConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'apps.user'

archivist/apps/user/forms.py

@@ -0,0 +1,13 @@
+from django import forms
+from django.contrib.auth.models import User
+from .models import UserProfile
+
+class UserProfileForm(forms.ModelForm):
+    class Meta:
+        model = UserProfile
+        fields = ['show_mature', "items_per_page", "post_seen_delay"]
+
+class UserForm(forms.ModelForm):
+    class Meta:
+        model = User
+        fields = ['username', 'email', 'first_name', 'last_name']

archivist/apps/user/migrations/0001_initial.py

@@ -0,0 +1,34 @@
+# Generated by Django 4.1.1 on 2023-11-01 14:43
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('furaffinity', '0001_initial'),
+        ('auth', '0012_alter_user_first_name_max_length'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='UserProfile',
+            fields=[
+                ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to=settings.AUTH_USER_MODEL)),
+                ('show_mature', models.CharField(choices=[('H', 'Hide'), ('B', 'Blur'), ('S', 'Show')], default='H', max_length=2)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='SeenPost',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('timestamp', models.DateTimeField(auto_now_add=True)),
+                ('post', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='furaffinity.fa_submission')),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='user.userprofile')),
+            ],
+        ),
+    ]

archivist/apps/user/models.py

@@ -0,0 +1,52 @@
+from django.db import models
+from django.contrib.auth.models import User
+
+from django.utils.translation import gettext_lazy as _
+
+from apps.sites.models import Submissions
+
+class UserProfile(models.Model):
+    
+    MATURE = [
+        ("H", "Hide"),
+        ("B", "Blur"),
+        ("S", "Show"),
+        ]
+    
+    ITEMS_PER_PAGE = [
+        (24, "24"),
+        (48, "48"),
+        (72, "72"),
+        ]
+    
+    POST_SEEN_DELAY = [
+        (15, "15"),
+        (30, "30"),
+        (60, "60"),
+        (90, "90"),
+        ]
+    
+    user = models.OneToOneField(User, on_delete=models.CASCADE, primary_key=True, unique=True)
+    show_mature = models.CharField(max_length=2, choices=MATURE, default=MATURE[0][0])
+    items_per_page = models.IntegerField(choices=ITEMS_PER_PAGE, default=ITEMS_PER_PAGE[0][0])
+    post_seen_delay = models.IntegerField(choices=POST_SEEN_DELAY, default=POST_SEEN_DELAY[1][1], help_text="Delay in seconds before marking a post as seen") 
+
+    class Meta:
+        verbose_name = _("User Profile")
+        verbose_name_plural = _("User Profiles")
+        
+    def __str__(self):
+        return self.user.username
+
+
+class SeenPost(models.Model):
+    user = models.ForeignKey(UserProfile, on_delete=models.CASCADE)
+    post = models.ForeignKey(Submissions, on_delete=models.CASCADE)
+    timestamp = models.DateTimeField(auto_now_add=True)
+
+    class Meta:
+        verbose_name = _("Seen Post")
+        verbose_name_plural = _("Seen Posts")
+    
+    def __str__(self) -> str:
+        return self.user.user.username + " - " + self.post.submission_hash

archivist/apps/user/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

archivist/apps/user/urls.py

@@ -0,0 +1,7 @@
+from django.urls import path
+from .views import ProfileEditView
+
+urlpatterns = [
+    # Other URL patterns
+    path('profile/edit', ProfileEditView, name='profile'),
+]

archivist/apps/user/views.py

@@ -0,0 +1,28 @@
+from django.shortcuts import render
+from django.contrib.auth.decorators import login_required
+
+from rest_framework import status
+from rest_framework.response import Response
+
+
+from apps.user.models import UserProfile, SeenPost
+
+from apps.sites.models import Submissions
+
+from .forms import UserProfileForm, UserForm
+
+@login_required(login_url="/login/")
+def ProfileEditView(request):
+    if request.method == 'POST':
+        user_form = UserForm(request.POST, instance=request.user)
+        profile_form = UserProfileForm(request.POST, instance=request.user.userprofile)
+
+        if user_form.is_valid() and profile_form.is_valid():
+            user_form.save()
+            profile_form.save()
+            # Redirect to a success page or home page
+    else:
+        user_form = UserForm(instance=request.user)
+        profile_form = UserProfileForm(instance=request.user.userprofile)
+
+    return render(request, 'accounts/profile.html', {'user_form': user_form, 'profile_form': profile_form})

archivist/core/__init__.py

@@ -0,0 +1,10 @@
+# core/__init__.py
+
+# Import the Celery app instance from the celery.py file in the same directory (core/celery.py).
+# The celery.py file is where the Celery app instance is created and configured.
+from .celery import app as celery_app
+
+# The __all__ variable is used to define what should be exported when someone imports the package (core).
+# By specifying __all__ = ("celery_app",), we are explicitly stating that the celery_app should be exported
+# when someone imports the core package.
+__all__ = ("celery_app",)

archivist/core/asgi.py

@@ -0,0 +1,16 @@
+"""
+ASGI config for archivist project.
+
+It exposes the ASGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/4.0/howto/deployment/asgi/
+"""
+
+import os
+
+from django.core.asgi import get_asgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings')
+
+application = get_asgi_application()

archivist/core/celery.py

@@ -0,0 +1,25 @@
+from __future__ import absolute_import, unicode_literals
+
+import os
+
+from celery import Celery
+
+# Set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings')
+
+app = Celery('archivist')
+
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object('django.conf:settings', namespace='CELERY')
+
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks()
+
+@app.task(bind=True)
+def debug_task(self):
+    print('Request: {0!r}'.format(self.request))
+
+

archivist/core/settings.py

@@ -0,0 +1,215 @@
+"""
+Django settings for archivist project.
+
+Generated by 'django-admin startproject' using Django 4.0.6.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/4.0/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/4.0/ref/settings/
+"""
+
+import os, environ
+
+env = environ.Env(
+    # set casting, default value
+    DEBUG=(bool, False)
+)
+
+# Build paths inside the project like this: BASE_DIR / 'subdir'.
+BASE_DIR = os.path.dirname(os.path.dirname(__file__))
+CORE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+# Take environment variables from .env file
+environ.Env.read_env(os.path.join(BASE_DIR, '.env'))
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/4.0/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = env('SECRET_KEY', default='S#perS3crEt_007')
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = env('DEBUG')
+
+# Assets Management
+ASSETS_ROOT = os.getenv('ASSETS_ROOT', '/static/assets') 
+
+# load production server from .env
+ALLOWED_HOSTS        = ['localhost', 'localhost:85', '127.0.0.1',               env('SERVER', default='127.0.0.1') ]
+CSRF_TRUSTED_ORIGINS = ['http://localhost:85', 'http://127.0.0.1', 'https://' + env('SERVER', default='127.0.0.1') ]
+
+
+
+# Application definition
+
+INSTALLED_APPS = [    
+    # Django core apps
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    
+    # Pip module libraries
+    'whitenoise.runserver_nostatic',
+    'rest_framework',
+    'sorl.thumbnail',
+    
+    'django_cleanup.apps.CleanupConfig',
+    
+    "django_celery_beat",
+    "django_celery_results",
+    
+    'apps.files',
+    
+    # Gallery Archivist apps    
+    'sites.furaffinity'
+]
+
+MIDDLEWARE = [
+    'django.middleware.security.SecurityMiddleware',
+    'whitenoise.middleware.WhiteNoiseMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+]
+
+ROOT_URLCONF = 'core.urls'
+TEMPLATE_DIR_APPS = os.path.join(CORE_DIR, "apps/templates")  # ROOT dir for templates
+TEMPLATE_DIR_SITES = os.path.join(CORE_DIR, "sites/templates")
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [TEMPLATE_DIR_APPS, TEMPLATE_DIR_SITES, "templates/",],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+                'apps.context_processors.debug',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'core.wsgi.application'
+
+#############################################################
+# Database
+# https://docs.djangoproject.com/en/4.0/ref/settings/#databases
+#############################################################
+if os.environ.get('DB_ENGINE') and os.environ.get('DB_ENGINE') == "mysql":
+    DATABASES = { 
+      'default': {
+        'ENGINE'  : 'django.db.backends.mysql', 
+        'NAME'    : os.getenv('DB_NAME'     , 'gallery_archivist_db'),
+        'USER'    : os.getenv('DB_USERNAME' , 'gallery_archivist_db_usr'),
+        'PASSWORD': os.getenv('DB_PASS'     , 'pass'),
+        'HOST'    : os.getenv('DB_HOST'     , 'localhost'),
+        'PORT'    : os.getenv('DB_PORT'     , 3306),
+        }, 
+    }
+else:
+    DATABASES = {
+        'default': {
+            'ENGINE': 'django.db.backends.sqlite3',
+            'NAME': 'db.sqlite3',
+        }
+    }
+#############################################################
+
+
+#############################################################
+# Password validation
+# https://docs.djangoproject.com/en/4.0/ref/settings/#auth-password-validators
+#############################################################
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+    },
+]
+#############################################################
+
+
+#############################################################
+# Internationalization
+# https://docs.djangoproject.com/en/4.0/topics/i18n/
+#############################################################
+LANGUAGE_CODE = 'en-us'
+
+TIME_ZONE = 'UTC'
+
+USE_I18N = True
+
+USE_TZ = True
+#############################################################
+
+
+#############################################################
+# SRC: https://devcenter.heroku.com/articles/django-assets
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/1.9/howto/static-files/
+#############################################################
+STATIC_ROOT = os.path.join(CORE_DIR, 'staticfiles')
+STATIC_URL = '/static/'
+
+# Extra places for collectstatic to find static files.
+STATICFILES_DIRS = (
+    os.path.join(CORE_DIR, 'apps/static'),
+    os.path.join(BASE_DIR, 'static'),
+)
+
+#STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage"
+
+MEDIA_URL = '/media/'
+MEDIA_ROOT = os.path.join(BASE_DIR, 'media')
+#############################################################
+
+
+#############################################################
+# Default primary key field type
+# https://docs.djangoproject.com/en/4.0/ref/settings/#default-auto-field
+#############################################################
+DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
+#############################################################
+
+
+#############################################################
+# Celery
+#############################################################
+CELERY_BROKER_URL = os.environ.get("CELERY_BROKER", 'redis://localhost:6379/0')
+
+CELERY_TIMEZONE = TIME_ZONE
+
+#CELERY_RESULT_BACKEND = "redis://localhost:6379/0"
+
+CELERY_RESULT_BACKEND = 'django-db'
+CELERY_CACHE_BACKEND = 'django-cache'
+
+CELERY_RESULT_EXTENDED = True
+
+CELERY_BEAT_SCHEDULER = 'django_celery_beat.schedulers:DatabaseScheduler'
+#############################################################
+
+
+# Allow embedding in iframes from the same origin
+X_FRAME_OPTIONS = 'SAMEORIGIN' 

archivist/core/templates/admin/base.html

@@ -0,0 +1,79 @@
+{% extends 'admin/base.html' %}
+
+{% block extrahead %}{{ block.super }}
+<style>
+
+@media (prefers-color-scheme: dark) {
+
+    :root {
+        --primary: #62095c; {% comment %} #9106c4; {% endcomment %}
+        --primary-fg: #eee;
+        --body-fg: #eeeeee;
+        --body-bg: #040b1e;
+        --body-quiet-color: #e0e0e0;
+        --body-loud-color: #ffffff;
+        --breadcrumbs-link-fg: #e0e0e0;
+        --breadcrumbs-bg: var(--primary);
+        --link-fg: #81d4fa;
+        --link-hover-color: #4ac1f7;
+        --link-selected-fg: #6f94c6;
+        --hairline-color: #272727;
+        --border-color: #353535;
+        --error-fg: #e35f5f;
+        --message-success-bg: #006b1b;
+        --message-warning-bg: #583305;
+        --message-error-bg: #570808;
+        --darkened-bg: #0f1e31;
+        --selected-bg: #1b1b1b;
+        --selected-row: #00363a;
+        --close-button-bg: #333333;
+        --close-button-hover-bg: #666666;
+        
+        --header-color: #fff;
+        --header-bg: #9106c4;
+    }
+}
+
+@media (prefers-color-scheme: light) {
+
+    :root {
+        --primary: #79aec8;
+        --secondary: #417690;
+        --accent: #f5dd5d;
+        --primary-fg: #fff;
+        --body-fg: #333;
+        --body-bg: #fff;
+        --body-quiet-color: #666;
+        --body-loud-color: #000;
+        --header-color: #ffc;
+        --header-branding-color: var(--accent);
+        --header-bg: var(--secondary);
+        --header-link-color: var(--primary-fg);
+        --breadcrumbs-fg: #c4dce8;
+        --breadcrumbs-link-fg: var(--body-bg);
+        --breadcrumbs-bg: var(--primary);
+        --link-fg: #447e9b;
+        --link-hover-color: #036;
+        --link-selected-fg: #5b80b2;
+        --hairline-color: #e8e8e8;
+        --border-color: #ccc;
+        --error-fg: #ba2121;
+        --message-success-bg: #dfd;
+        --message-warning-bg: #ffc;
+        --message-error-bg: #ffefef;
+        --darkened-bg: #f8f8f8;
+        --selected-bg: #e4e4e4;
+        --selected-row: #ffc;
+        --button-fg: #fff;
+        --button-bg: var(--primary);
+        --button-hover-bg: #609ab6;
+        --default-button-bg: var(--secondary);
+        --default-button-hover-bg: #205067;
+        --close-button-bg: #888;
+        --close-button-hover-bg: #747474;
+        --delete-button-bg: #ba2121;
+        --delete-button-hover-bg: #a41515;
+  }
+}
+</style>
+{% endblock %}

archivist/core/urls.py

@@ -0,0 +1,29 @@
+"""archivist URL Configuration
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+    https://docs.djangoproject.com/en/4.0/topics/http/urls/
+Examples:
+Function views
+    1. Add an import:  from my_app import views
+    2. Add a URL to urlpatterns:  path('', views.home, name='home')
+Class-based views
+    1. Add an import:  from other_app.views import Home
+    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
+Including another URLconf
+    1. Import the include() function: from django.urls import include, path
+    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
+"""
+from django.contrib import admin
+from django.urls import path, include
+
+admin.site.site_header = 'Gallery Archivist Django Admin Panel'
+
+urlpatterns = [
+    path('admin/', admin.site.urls),    # Django admin route
+    path("", include("apps.authentication.urls")),    # Auth routes - login / register
+    path("files/", include("apps.files.urls")),
+    path("sites/", include("apps.sites.urls")),
+    
+    path("", include("sites.furaffinity.urls")),
+    
+]

archivist/core/wsgi.py

@@ -0,0 +1,16 @@
+"""
+WSGI config for archivist project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/4.0/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings')
+
+application = get_wsgi_application()

archivist/manage.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+"""Django's command-line utility for administrative tasks."""
+import os
+import sys
+
+
+def main():
+    """Run administrative tasks."""
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings')
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError as exc:
+        raise ImportError(
+            "Couldn't import Django. Are you sure it's installed and "
+            "available on your PYTHONPATH environment variable? Did you "
+            "forget to activate a virtual environment?"
+        ) from exc
+    execute_from_command_line(sys.argv)
+
+
+if __name__ == '__main__':
+    main()

archivist/sites/furaffinity/__init__.py

@@ -0,0 +1,2 @@
+# __init__.py
+

archivist/sites/furaffinity/admin.py

@@ -0,0 +1,32 @@
+from django.contrib import admin
+from .models import FA_Submission, FA_Submission_File, FA_Tags, FA_User, FA_Species, FA_Gender
+
+# Register your models here.
+
+class FA_UserAdmin(admin.ModelAdmin):
+    fieldsets = (
+        ("Artist Name", { "fields": ["artist", "artist_url"], } ),
+    )
+
+
+class FA_TagsAdmin(admin.ModelAdmin):
+    
+    @admin.display(description='Tag Name')
+    def upper_case_tag(obj):
+        return ("%s" % (obj.tag).capitalize())
+    
+    list_display = (upper_case_tag,)
+
+
+class FA_SubmissionAdmin(admin.ModelAdmin):
+    list_display = ('submission_id', 'title', 'artist', 'date', 'date_added', 'mature_rating',)
+    
+class FA_Submission_FileAdmin(admin.ModelAdmin):
+    list_display = ('file_name', 'date_added', 'file_hash', 'file',)
+
+admin.site.register(FA_User, FA_UserAdmin,)
+admin.site.register(FA_Tags, FA_TagsAdmin,)
+admin.site.register(FA_Species,)
+admin.site.register(FA_Gender,)
+admin.site.register(FA_Submission, FA_SubmissionAdmin,)
+admin.site.register(FA_Submission_File, FA_Submission_FileAdmin,)

archivist/sites/furaffinity/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class FuraffinityConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'sites.furaffinity'

archivist/sites/furaffinity/config.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class MyConfig(AppConfig):
+    name = 'sites.furaffinity'
+    label = 'sites_furaffinity'

archivist/sites/furaffinity/forms.py

@@ -0,0 +1,15 @@
+from django import forms
+
+from .models import FA_Submission
+
+class SearchForm(forms.Form):
+    search_query = forms.CharField(label='Search', max_length=100)
+
+class URLImportForm(forms.Form):
+    url = forms.URLField(label='Post URL', required=True)
+
+
+#class DateTimeForm(forms.ModelForm):
+#    class Meta:
+#        model = FA_Submission
+#        fields = ['submission_id', 'media_url', 'title', 'description', 'artist', 'date', 'species']

archivist/sites/furaffinity/management/commands/import_data.py

@@ -0,0 +1,186 @@
+# your_app/management/commands/import_data.py
+
+import os
+import json
+from blake3 import blake3
+from tqdm.auto import tqdm
+
+#from PIL import Image
+from datetime import datetime
+
+from django.core.management.base import BaseCommand
+from django.utils.text import slugify
+from django.core.exceptions import ObjectDoesNotExist
+#from django.core.files import File
+from django.utils import timezone
+
+from sites.furaffinity.models import FA_Submission, FA_Submission_File, FA_Tags, FA_User, FA_Species, FA_Gender, FA_Mature
+
+class Command(BaseCommand):
+    help = 'Import data from JSON files in a folder or a single JSON file to the furaffinity archive'
+
+    def add_arguments(self, parser):
+        parser.add_argument('path', type=str, help='Path to the folder containing JSON files or a single JSON file')
+        parser.add_argument('--delete', action='store_true', help='Delete imported files')
+
+
+    def handle(self, *args, **kwargs):
+        path = kwargs['path']
+        delete = kwargs['delete']
+
+        if os.path.isfile(path):
+            self.process_json_file(path, delete)
+        elif os.path.isdir(path):
+            self.process_json_folder(path, delete)
+        else:
+            self.stdout.write(self.style.ERROR(f"The path '{path}' is not a valid file or folder."))
+            return
+
+
+    def process_json_file(self, file_path, delete):
+        #self.stdout.write(self.style.NOTICE(f"Importing data from: {file_path}"))
+        tqdm.write(self.style.NOTICE(f"Importing data from: {file_path}"))
+
+        with open(file_path) as f:
+            data = json.load(f)
+
+        self.import_data(data, file_path, delete)
+
+        #self.stdout.write(self.style.SUCCESS('Data imported successfully.'))
+        tqdm.write(self.style.SUCCESS('Data imported successfully.'))
+    
+    def process_json_folder(self, folder_path, delete):
+        if not os.path.exists(folder_path):
+            #self.stdout.write(self.style.ERROR(f"The folder '{folder_path}' does not exist."))
+            tqdm.write(self.style.ERROR(f"The folder '{folder_path}' does not exist."))
+            return
+
+        for root, dirs, files in tqdm(os.walk(folder_path), dynamic_ncols=True):
+            for file_name in files:
+                if file_name.endswith('.json'):
+                    file_path = os.path.join(root, file_name)
+                    self.process_json_file(file_path, delete)
+                    
+
+
+    def import_data(self, data, json_file_path, delete):    
+        #self.stdout.write(self.style.NOTICE(data))
+        
+        submission, created = FA_Submission.objects.get_or_create(submission_id=data["id"])
+        
+        submission.media_url     = data["url"]
+        submission.title         = data["title"]
+        submission.description   = data["description"]
+        
+        submission.date = timezone.make_aware(datetime.strptime(data["date"], "%Y-%m-%d %H:%M:%S"))
+        
+        artist, created = FA_User.objects.get_or_create(artist_url=data["artist_url"], artist=data["artist"])
+        submission.artist = artist
+        
+        species, created = FA_Species.objects.get_or_create(species=data["species"])
+        submission.species = species
+        
+        for tag_name in data["tags"]:
+            tag_slug = slugify(tag_name)
+            try:
+                # Check if the tag already exists in the database by name
+                tag = FA_Tags.objects.get(tag_slug=tag_slug)
+            except ObjectDoesNotExist:
+                # If the tag does not exist, create a new tag and generate the slug
+                tag = FA_Tags(tag=tag_name)
+                tag.tag_slug = tag_slug
+
+            tag.save()  # Save the tag (either new or existing)
+
+            submission.tags.add(tag)  # Add the tag to the submission
+        
+        mature, created = FA_Mature.objects.get_or_create(mature=data["rating"])
+        submission.mature_rating = mature
+        
+        submission.number_of_comments = data["comments"]
+        submission.views = data["views"]
+        
+        gender, created = FA_Gender.objects.get_or_create(gender=data["gender"])
+        submission.gender = gender
+        
+        submission.fa_theme = data["theme"]
+        submission.fa_category = data["fa_category"]
+        submission.gallery_type = data["subcategory"]
+        submission.file_extension = data["extension"]
+        submission.image_height = data["height"]
+        submission.image_width = data["width"]
+        
+        
+        file_path = json_file_path.removesuffix(".json")
+        
+        # Handle file import
+        if os.path.exists(file_path):
+            file_hash = self.compute_file_hash(file_path)
+            try:
+                file_instance = FA_Submission_File.objects.get(file_hash=file_hash)
+                #self.stdout.write(self.style.NOTICE(f"Skipping: {file_path} file, already imported"))
+                tqdm.write(self.style.NOTICE(f"Skipping: {file_path} file, already imported"))
+            except FA_Submission_File.DoesNotExist:
+                # If the file doesn't exist, create a new file instance and link it to the submission
+                with open(file_path, 'rb') as file:
+                    file_instance = FA_Submission_File()
+                    file_instance.file_hash = file_hash
+                    
+                    file_name = os.path.basename(file_path)
+                    Null, file_ext = os.path.splitext(file_name)
+                    hash_file_name = file_hash + file_ext
+                    file_instance.file.save(hash_file_name, file)
+                    
+                    file_instance.file_name = file_name
+                    file_instance.save()
+                    
+                    tqdm.write(self.style.NOTICE(f"Import media file: {file_path}"))
+            
+            # Now link the image_instance to your_model_instance
+            submission.file = file_instance
+            
+            
+        else:
+            #self.stdout.write(self.style.WARNING(f"File not found: {file_path}"))
+            tqdm.write(self.style.WARNING(f"File not found: {file_path}"))   
+                #file, created = FA_Submission_File.objects.get_or_create(file=file_hash)
+                #print(file)
+                #if not FA_Submission.objects.filter(image=image_hash).exists():
+                #    with open(image_file_path, 'rb') as img_file:
+                #        submission.image.save(os.path.basename(image_file_path), File(img_file), save=True)
+                #    submission.image_hash = image_hash  # Save the image hash in the model
+                #else:
+                #    self.stdout.write(self.style.WARNING(f"Skipping duplicate image: {image_file_path}"))
+                    
+        #if os.path.exists(image_file_path):
+        #    with open(image_file_path, 'rb') as img_file:
+        #        submission.image.save(os.path.basename(image_file_path), File(img_file), save=True)
+        
+        submission.save()
+
+        self.delete_imported_file(json_file_path, delete)
+        self.delete_imported_file(file_path, delete)
+
+
+    def compute_file_hash(self, file_path):
+        try:
+            # Compute BLAKE3 hash of the file
+            hasher = blake3()
+            with open(file_path, 'rb') as f:
+                while chunk := f.read(65536):
+                    hasher.update(chunk)
+            return hasher.hexdigest()
+        except Exception as e:
+            #self.stdout.write(self.style.WARNING(f"Error computing file hash: {e}"))
+            tqdm.write(self.style.WARNING(f"Error computing file hash: {e}"))
+            return None
+
+
+    # Delete the file if the --delete flag is used
+    def delete_imported_file(self, file_path, delete):
+        if delete:
+            if os.path.exists(file_path):
+                os.remove(file_path)
+                tqdm.write(self.style.SUCCESS(f"Deleted: {file_path}"))
+            else:
+                tqdm.write(self.style.WARNING(f"File not found: {file_path}"))

archivist/sites/furaffinity/migrations/0001_initial.py

@@ -0,0 +1,131 @@
+# Generated by Django 4.1.1 on 2023-10-21 16:51
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='FA_Gender',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('gender', models.CharField(max_length=30, unique=True)),
+            ],
+            options={
+                'verbose_name': 'Gender',
+                'verbose_name_plural': 'Genders',
+            },
+        ),
+        migrations.CreateModel(
+            name='FA_Mature',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('mature', models.CharField(max_length=30, unique=True)),
+            ],
+            options={
+                'verbose_name': 'Mature Rating',
+                'verbose_name_plural': 'Mature Ratings',
+            },
+        ),
+        migrations.CreateModel(
+            name='FA_Species',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('species', models.CharField(max_length=50, unique=True)),
+            ],
+            options={
+                'verbose_name': 'Species',
+                'verbose_name_plural': 'Species',
+            },
+        ),
+        migrations.CreateModel(
+            name='FA_Submission_File',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('file_hash', models.CharField(max_length=64, unique=True)),
+                ('file_name', models.CharField(blank=True, max_length=150)),
+                ('file', models.FileField(blank=True, upload_to='furaffinity/submissions/')),
+                ('date_added', models.DateTimeField(auto_now_add=True)),
+            ],
+            options={
+                'verbose_name': 'Submission File',
+                'verbose_name_plural': 'Submission Files',
+            },
+        ),
+        migrations.CreateModel(
+            name='FA_Tags',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('tag', models.CharField(max_length=250, unique=True, verbose_name='Tag name')),
+                ('tag_slug', models.SlugField(max_length=260, unique=True, verbose_name='Tag slug/url')),
+            ],
+            options={
+                'verbose_name': 'Tag',
+                'verbose_name_plural': 'Tags',
+            },
+        ),
+        migrations.CreateModel(
+            name='FA_UserIconFile',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('icon_file_hash', models.CharField(max_length=64, unique=True)),
+                ('icon_file_name', models.CharField(blank=True, max_length=150)),
+                ('icon_file', models.FileField(blank=True, upload_to='furaffinity/user_icon/')),
+                ('date_added', models.DateTimeField(auto_now_add=True)),
+            ],
+            options={
+                'verbose_name': 'User Icon File',
+                'verbose_name_plural': 'User Icon Files',
+            },
+        ),
+        migrations.CreateModel(
+            name='FA_User',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('artist', models.CharField(max_length=35)),
+                ('artist_url', models.SlugField(max_length=40, unique=True)),
+                ('description', models.TextField(blank=True)),
+                ('icon', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='furaffinity.fa_usericonfile')),
+            ],
+            options={
+                'verbose_name': 'User',
+                'verbose_name_plural': 'Users',
+            },
+        ),
+        migrations.CreateModel(
+            name='FA_Submission',
+            fields=[
+                ('submission_id', models.PositiveBigIntegerField(default=1, primary_key=True, serialize=False, unique=True, verbose_name='ID')),
+                ('media_url', models.URLField(blank=True)),
+                ('title', models.CharField(default='', max_length=60)),
+                ('description', models.TextField(blank=True, default='', null=True)),
+                ('date', models.DateTimeField(null=True)),
+                ('date_added', models.DateTimeField(auto_now_add=True)),
+                ('number_of_comments', models.PositiveIntegerField(null=True)),
+                ('views', models.PositiveIntegerField(null=True)),
+                ('fa_theme', models.CharField(max_length=50, null=True, verbose_name='FA Theme')),
+                ('fa_category', models.CharField(max_length=50, null=True, verbose_name='FA Category')),
+                ('gallery_type', models.CharField(blank=True, max_length=20)),
+                ('file_extension', models.CharField(blank=True, max_length=10)),
+                ('image_height', models.PositiveIntegerField(blank=True, null=True)),
+                ('image_width', models.PositiveIntegerField(blank=True, null=True)),
+                ('artist', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='submissions', to='furaffinity.fa_user')),
+                ('file', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='furaffinity.fa_submission_file')),
+                ('gender', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='furaffinity.fa_gender')),
+                ('mature_rating', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='furaffinity.fa_mature')),
+                ('species', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='furaffinity.fa_species')),
+                ('tags', models.ManyToManyField(to='furaffinity.fa_tags')),
+            ],
+            options={
+                'verbose_name': 'Submission',
+                'verbose_name_plural': 'Submissions',
+            },
+        ),
+    ]

archivist/sites/furaffinity/models.py

@@ -0,0 +1,222 @@
+from django.utils.translation import gettext_lazy as _
+from django.utils.text import slugify
+from django.urls import reverse
+
+from django.db import models
+
+import os
+
+# Create your models here.
+
+class FA_UserIconFile(models.Model):
+
+    icon_file_hash  = models.CharField(unique=True, max_length=64,)
+    icon_file_name  = models.CharField(max_length=150, blank=True)
+    icon_file       = models.FileField(upload_to="furaffinity/user_icon/", blank=True)
+    date_added      = models.DateTimeField(auto_now_add=True, editable=True)    
+
+    class Meta:
+        verbose_name = _("User Icon File")
+        verbose_name_plural = _("User Icon Files")
+
+    def __str__(self):
+        return self.name
+
+    #def get_absolute_url(self):
+    #    return reverse("FA_UserIconFile_detail", kwargs={"pk": self.pk})
+
+
+class FA_User(models.Model):
+
+    artist      = models.CharField(max_length=35, unique=False,)
+    artist_url  = models.SlugField(max_length=40, unique=True,)
+    description = models.TextField(blank=True)
+    icon        = models.ForeignKey(FA_UserIconFile, on_delete=models.CASCADE, blank=True, null=True)
+    
+
+    def __unicode__(self):
+        return self.word
+
+    class Meta:
+        verbose_name = _("User")
+        verbose_name_plural = _("Users")
+
+    def __str__(self):
+        return self.artist
+
+    def get_absolute_url(self):
+        return f"/fa/user/{self.artist_url}"
+        #return reverse("fa_user", kwargs={"pk": self.pk})
+
+
+class FA_Tags(models.Model):
+    
+    tag      = models.CharField(verbose_name="Tag name", max_length=250, unique=True)
+    tag_slug = models.SlugField(verbose_name="Tag slug/url", max_length=260, unique=True)
+
+    class Meta:
+        verbose_name = _("Tag")
+        verbose_name_plural = _("Tags")
+
+    def save(self, *args, **kwargs):
+    # add slug if not already exists on save.
+        if not self.tag_slug:
+            self.tag_slug = slugify(self.tag)
+        super().save(*args, **kwargs)
+    
+    def __str__(self):
+        return self.tag
+    
+    def get_absolute_url(self):
+        return f"/fa/tag/{self.tag_slug}"
+
+#    def get_absolute_url(self):
+#        return f"/fa/tag/{self.tag_url}"
+#        return reverse("FA_Tag_detail", kwargs={"pk": self.pk})
+
+class FA_Species(models.Model):
+
+    species = models.CharField(unique=True, max_length=50)
+
+    class Meta:
+        verbose_name = _("Species")
+        verbose_name_plural = _("Species")
+
+    def __str__(self):
+        return self.species
+
+#    def get_absolute_url(self):
+#        return reverse("FA_Species_detail", kwargs={"pk": self.pk})
+
+class FA_Gender(models.Model):
+
+    gender = models.CharField(unique=True, max_length=30)
+
+    class Meta:
+        verbose_name = _("Gender")
+        verbose_name_plural = _("Genders")
+
+    def __str__(self):
+        return self.gender
+
+#    def get_absolute_url(self):
+#        return reverse("FA_Gender_detail", kwargs={"pk": self.pk})
+
+class FA_Mature(models.Model):
+
+    mature = models.CharField(max_length=30, unique=True) #verbose_name="Mature Rating",
+
+    class Meta:
+        verbose_name = _("Mature Rating")
+        verbose_name_plural = _("Mature Ratings")
+
+    def __str__(self):
+        return self.mature
+
+    #def get_absolute_url(self):
+    #    return reverse("FA_Mature_detail", kwargs={"pk": self.pk})
+
+class FA_Submission_File(models.Model):
+
+    file_hash  = models.CharField(unique=True, max_length=64,)
+    file_name  = models.CharField(max_length=150, blank=True)
+    file       = models.FileField(upload_to="furaffinity/submissions/", blank=True)
+    date_added = models.DateTimeField(auto_now_add=True, editable=True)
+
+    class Meta:
+        verbose_name = _("Submission File")
+        verbose_name_plural = _("Submission Files")
+
+    def __str__(self):
+        return self.file_hash
+
+    #def delete(self, *args, **kwargs):
+    #    # Delete the file from the disk when the model instance is deleted
+    #    print('Deleting image:', self.file)
+    #    if self.file:
+    #        if os.path.isfile(self.file.path):
+    #            os.remove(self.file.path)
+    #
+    #    super().delete(*args, **kwargs)
+
+    #def get_absolute_url(self):
+    #    return reverse("Submission_Image_detail", kwargs={"pk": self.pk})
+
+
+
+class FA_Submission(models.Model):
+
+    MATURE = [
+        ("G", "General"),
+        ("M", "Mature"),
+        ("A", "Adult"),
+        ]
+    
+    GENDER = [
+        ("Ma", "Male"),
+        ("Fe", "Female"),
+        ("He", "Herm"),
+        ("In", "Intersex"),
+        ("TM", "Trans (Male)"),
+        ("TF", "Trans (Female)"),
+        ("NB", "Non-Binary"),
+        ("MC", "Multiple characters"), 
+        ("ON", "Other / Not Specified"),
+        ] 
+
+    # id
+    submission_id      = models.PositiveBigIntegerField(verbose_name="ID", primary_key=True, unique=True, default=1)
+    # url
+    media_url          = models.URLField(blank=True)
+    # title
+    title              = models.CharField(default="", max_length=60)
+    # description
+    description        = models.TextField(default="", null=True, blank=True)
+    # artist
+    artist             = models.ForeignKey(FA_User, on_delete=models.CASCADE, null=True, related_name='submissions')
+    # date
+    date               = models.DateTimeField(null=True)
+    # NOT imported
+    date_added         = models.DateTimeField(auto_now_add=True, editable=True)
+    # species
+    species            = models.ForeignKey(FA_Species, on_delete=models.CASCADE, null=True)
+    # tags
+    tags               = models.ManyToManyField(FA_Tags)
+    # rating
+    mature_rating      = models.ForeignKey(FA_Mature, on_delete=models.CASCADE, null=True) #CharField(max_length=2, choices=MATURE, default=MATURE[0][0])
+    # comments
+    number_of_comments = models.PositiveIntegerField(null=True)
+    # views
+    views              = models.PositiveIntegerField(null=True)
+    # gender
+    gender             = models.ForeignKey(FA_Gender, on_delete=models.CASCADE, null=True)
+    # theme
+    fa_theme           = models.CharField(verbose_name="FA Theme", max_length=50, null=True)
+    # fa_category
+    fa_category        = models.CharField(verbose_name="FA Category", max_length=50, null=True)
+    # subcategory
+    gallery_type       = models.CharField(max_length=20, blank=True)
+    # extension
+    file_extension     = models.CharField(max_length=10, blank=True)
+    # height
+    image_height       = models.PositiveIntegerField(blank=True, null=True)
+    # width
+    image_width        = models.PositiveIntegerField(blank=True, null=True)
+    # ImageField for the image files
+    file               = models.ForeignKey(FA_Submission_File, on_delete=models.CASCADE, blank=True, null=True)  
+    
+    
+    class Meta:
+        verbose_name = _("Submission")
+        verbose_name_plural = _("Submissions")
+
+    def __str__(self):
+        return self.title
+    
+    #def save(self, *args, **kwargs):
+    #    if not self.related_model:
+    #        self.related_model, created = FA_User.objects.get_or_create(name=self.artist)
+    #    super().save(*args, **kwargs)
+
+#    def get_absolute_url(self):
+#        return reverse("FA_Submission_detail", kwargs={"pk": self.pk})

archivist/sites/furaffinity/serializers.py

@@ -0,0 +1,14 @@
+from django.contrib.auth.models import User, Group
+from rest_framework import serializers
+
+
+class UserSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = User
+        fields = ['url', 'username', 'email', 'groups']
+
+
+class GroupSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = Group
+        fields = ['url', 'name']

archivist/sites/furaffinity/tasks.py

@@ -0,0 +1,153 @@
+#tasks.py
+import requests
+import subprocess, os
+
+from sys import stdout, stderr 
+
+from bs4 import BeautifulSoup
+from blake3 import blake3
+
+from celery import shared_task
+
+from .models import FA_User, FA_UserIconFile
+
+@shared_task
+def fa_import_data():
+    try:
+        # Get the current working directory
+#        current_dir = os.getcwd()
+
+        # Change the working directory to the parent folder
+#        os.chdir(os.path.dirname(current_dir))
+        
+        result = subprocess.run(["python", "manage.py", "import_data", "gallery-dl/", "--delete"], capture_output=True, text=True)
+        
+        return {
+            'stdout': result.stdout,
+            'stderr': result.stderr,
+            'returncode': result.returncode
+        }
+    except Exception as e:
+        return {'error': str(e)}
+
+
+@shared_task
+def scrape_fa_submission(url):
+#    print(url)
+#    print(subprocess.run(['pwd'], capture_output=True, text=True))
+    try:
+        # Get the current working directory
+#        current_dir = os.getcwd()
+
+        # Change the working directory to the parent folder
+#        os.chdir(os.path.dirname(current_dir))
+        
+        
+        result = subprocess.run(['gallery-dl', '-c','../gallery-dl.conf', '-d', 'gallery-dl', '--mtime-from-date', '--write-metadata', url], capture_output=True, text=True)
+                
+        importTask = fa_import_data.delay()
+        print(importTask)
+        
+        return {
+            'stdout': result.stdout,
+            'stderr': result.stderr,
+            'returncode': result.returncode
+        }
+    except Exception as e:
+        return {'error': str(e)}
+
+
+@shared_task
+def get_fa_user_info(user):
+    
+    #try:
+    #    file_instance = FA_UserIconFile.objects.get(file_hash=file_hash)
+    #    #self.stdout.write(self.style.NOTICE(f"Skipping: {file_path} file, already imported"))
+    #    print(f"Skipping: {file_path} file, already imported")
+    #except FA_UserIconFile.DoesNotExist:
+    #    # If the file doesn't exist, create a new file instance and link it to the submission
+    #    with open(file_path, 'rb') as file:
+    #        file_instance = FA_Submission_File()
+    #        file_instance.file_hash = file_hash
+    #        
+    #        file_name = os.path.basename(file_path)
+    #        Null, file_ext = os.path.splitext(file_name)
+    #        hash_file_name = file_hash + file_ext
+    #        file_instance.file.save(hash_file_name, file)
+    #        
+    #        file_instance.file_name = file_name
+    #        file_instance.save()
+    
+#    # Now link the image_instance to your_model_instance
+#    submission.file = file_instance
+    
+    url = "https://www.furaffinity.net/user/" + user  # Replace with the URL of the page you want to scrape
+
+    # Fetch the web page content
+    response = requests.get(url)
+    if response.status_code == 200:
+        html_content = response.text
+    else:
+        return f"Error: Unable to fetch the page. Status code: {response.status_code}"
+    
+    # Parse the HTML content using BeautifulSoup
+    soup = BeautifulSoup(html_content, "html.parser")
+    
+    """ title = soup.title.text """
+    # Find the 'img' tag inside the 'a' tag with class 'current' inside the 'userpage-nav-avatar' tag
+    img_tag = soup.select_one('userpage-nav-avatar a.current img')
+    
+    if img_tag:
+        # Extract the 'src' attribute of the 'img' tag to get the image URL
+        image_url = img_tag['src']
+        return image_url
+    else:
+        stderr.write("Image not found on the page.")
+
+    #return title
+
+def compute_file_hash(self, file):
+        try:
+            # Compute BLAKE3 hash of the file
+            hasher = blake3()
+            with open(file, 'rb') as f:
+                while chunk := f.read(65536):
+                    hasher.update(chunk)
+            return hasher.hexdigest()
+        except Exception as e:
+            self.stdout.write(self.style.WARNING(f"Error computing file hash: {e}"))
+            return None
+
+
+
+def save_image_to_model(image_url):
+    try:
+        # Download the image from the URL
+        response = requests.get(image_url)
+        response.raise_for_status()
+
+        # Create a new instance of YourModel
+        instance = F()
+
+        # Save the image to the FileField
+        instance.image_field.save(f'image_{instance.pk}.jpg', ContentFile(response.content), save=True)
+
+        # Save the model instance to the database
+        instance.save()
+
+        return instance
+    except requests.exceptions.RequestException as e:
+        print(f"Failed to download the image: {e}")
+
+    return None
+
+@shared_task
+def test_task():
+    print("This is a test task. Celery is working!")
+    num = 12 * 2
+    return num
+
+@shared_task
+def calculate_square(number1, number2):
+    result = number1 * number2
+    return result