From 7165605f5e2316f5d008c618e5f4e0600e0ff843 Mon Sep 17 00:00:00 2001
From: Aroy-Art <Aroy-Art@pm.me>
Date: Sun, 16 Mar 2025 22:14:40 +0100
Subject: [PATCH] Add: furaffinity data importer to import data command
---
.../management/commands/import_data.py | 158 ++++++++++++++++++
1 file changed, 158 insertions(+)
diff --git a/backend/apps/archive/management/commands/import_data.py b/backend/apps/archive/management/commands/import_data.py
index dc52454..8d4c988 100644
--- a/backend/apps/archive/management/commands/import_data.py
+++ b/backend/apps/archive/management/commands/import_data.py
@@ -395,31 +395,188 @@ class TwitterImporter(BaseImporter):
post_instance.save()
+
+class FurAffinityImporter(BaseImporter):
+ """Importer for FurAffinity data."""
+
+ def import_data(
+ self, data: Dict[str, Any], file_path_json: str, delete: bool
+ ) -> None:
+ """Import FurAffinity data from JSON into the database."""
+ try:
+ category = data.get("category", "furaffinity")
+ source_site_instance = self.get_or_create_source_site(category)
+
+ # Process creator
+ creator_instance = self._process_creator(data, source_site_instance)
+
+ # Process category
+ category_instance = self._process_category(data)
+
+ # Process post
+ self._process_post(
+ data,
+ source_site_instance,
+ creator_instance,
+ category_instance,
+ file_path_json,
+ delete,
)
+ except Exception as e:
+ self.log_error(f"Error importing FurAffinity data: {str(e)}")
+ def _process_creator(self, data, source_site_instance):
+ """Process creator data for FurAffinity."""
+ artist = data.get("artist", "")
+ artist_url = data.get("artist_url", artist.lower())
+ creator_instance, _ = CreatorModel.objects.get_or_create(
+ slug=artist_url, source_site=source_site_instance
+ )
+
+ creator_instance.name = artist
+ creator_instance.creator_id = artist_url
+
+ # We don't have creator creation date in FurAffinity data
+ # Using post date as an approximation
+ if "date" in data:
+ creator_instance.date_created = timezone.make_aware(
+ datetime.strptime(data["date"], "%Y-%m-%d %H:%M:%S")
)
creator_instance.save()
+ return creator_instance
+
+ def _process_category(self, data):
+ """Process category data for FurAffinity."""
+ subcategory = data.get("subcategory", "gallery")
+
+ category_instance, created = CategoryModel.objects.get_or_create(
+ slug=subcategory
)
+ if created:
+ category_instance.name = subcategory.capitalize()
+ # Process FA-specific categories
+ if "fa_category" in data:
+ fa_category = data["fa_category"]
+ fa_category_instance, _ = CategoryModel.objects.get_or_create(
+ slug=fa_category.lower().replace(" ", "_")
+ )
+ fa_category_instance.name = fa_category
+ fa_category_instance.save()
+ category_instance.save()
+ return category_instance
+ def _process_post(
+ self,
+ data,
+ source_site_instance,
+ creator_instance,
+ category_instance,
+ file_path_json,
+ delete,
+ ):
+ """Process post data for FurAffinity."""
+ post_id = str(data.get("id", ""))
+ post_instance, _ = PostModel.objects.get_or_create(
+ post_id=post_id, source_site=source_site_instance
+ )
+ # Add category
+ if category_instance:
+ post_instance.category.add(category_instance)
+
+ # Add category to creator
+ if creator_instance:
+ creator_instance.refresh_from_db()
+ creator_instance.categories.add(category_instance)
+ creator_instance.save()
+
+ # Link creator
+ if creator_instance:
+ post_instance.creator = creator_instance
+
+ # Set creation date
+ if "date" in data:
+ post_instance.date_created = timezone.make_aware(
+ datetime.strptime(data["date"], "%Y-%m-%d %H:%M:%S")
+ )
+
+ # Set mature content flag based on rating
+ rating = data.get("rating", "").lower()
+ post_instance.mature = rating in ["mature", "adult"]
+
+ # Add description (title + description)
+ title = data.get("title", "")
+ description = data.get("description", "")
+
+ full_description = f"{title}\n\n{description}" if title else description
+
+ if full_description:
+ self.add_description(
+ description_text=full_description,
+ date_str=data["date"],
+ date_format="%Y-%m-%d %H:%M:%S",
+ owner_instance=post_instance,
+ owner_type="post",
+ file_date=os.path.getmtime(file_path_json),
+ )
+
+ # Add tags
+ if "tags" in data:
+ self.add_tags(data["tags"], post_instance)
+
+ # Add species as a special tag if present
+ if "species" in data and data["species"] not in [
+ "Unspecified / Any",
+ "Any",
+ ]:
+ species_tags = [s.strip() for s in data["species"].split("/")]
+ self.add_tags(species_tags, post_instance)
+
+ # Add gender as a special tag if present
+ if "gender" in data and data["gender"] not in ["Unspecified / Any", "Any"]:
+ gender_tags = [g.strip() for g in data["gender"].split("/")]
+ self.add_tags(gender_tags, post_instance)
+
+ # Add metadata as JSON field if your model supports it
+ metadata = {}
+
+ for field in ["views", "favorites", "comments", "theme", "fa_category"]:
+ if field in data:
+ metadata[field] = data[field]
+
+ # If your PostModel has a metadata JSONField, uncomment this
+ # post_instance.metadata = metadata
+
+ # Import the file
file_path = file_path_json.removesuffix(".json")
+ # Check if the file exists, otherwise try to construct from filename and extension
+ if not os.path.exists(file_path) and "filename" in data and "extension" in data:
+ alt_file_path = f"{os.path.dirname(file_path_json)}/{data['filename']}.{data['extension']}"
+ file_instance = self.import_file(alt_file_path, delete)
+ else:
+ file_instance = self.import_file(file_path, delete)
if file_instance:
post_instance.files.add(file_instance)
+ # Add known image dimensions if available
+ if not file_instance.width and "width" in data:
+ file_instance.width = data.get("width")
if not file_instance.height and "height" in data:
file_instance.height = data.get("height")
if "width" in data or "height" in data:
+ file_instance.save()
+
post_instance.save()
@@ -432,6 +589,7 @@ class Command(BaseCommand):
super().__init__(*args, **kwargs)
self.importers = {
"twitter": TwitterImporter(self),
+ "furaffinity": FurAffinityImporter(self),
}
# Set up logging