Add: furaffinity data importer to import data command

This commit is contained in:
Aroy-Art 2025-03-16 22:14:40 +01:00
parent 72f3a9b7b1
commit 7165605f5e
Signed by: Aroy
GPG key ID: 583642324A1D2070

View file

@ -395,31 +395,188 @@ class TwitterImporter(BaseImporter):
post_instance.save()
class FurAffinityImporter(BaseImporter):
"""Importer for FurAffinity data."""
def import_data(
self, data: Dict[str, Any], file_path_json: str, delete: bool
) -> None:
"""Import FurAffinity data from JSON into the database."""
try:
category = data.get("category", "furaffinity")
source_site_instance = self.get_or_create_source_site(category)
# Process creator
creator_instance = self._process_creator(data, source_site_instance)
# Process category
category_instance = self._process_category(data)
# Process post
self._process_post(
data,
source_site_instance,
creator_instance,
category_instance,
file_path_json,
delete,
)
except Exception as e:
self.log_error(f"Error importing FurAffinity data: {str(e)}")
def _process_creator(self, data, source_site_instance):
"""Process creator data for FurAffinity."""
artist = data.get("artist", "")
artist_url = data.get("artist_url", artist.lower())
creator_instance, _ = CreatorModel.objects.get_or_create(
slug=artist_url, source_site=source_site_instance
)
creator_instance.name = artist
creator_instance.creator_id = artist_url
# We don't have creator creation date in FurAffinity data
# Using post date as an approximation
if "date" in data:
creator_instance.date_created = timezone.make_aware(
datetime.strptime(data["date"], "%Y-%m-%d %H:%M:%S")
)
creator_instance.save()
return creator_instance
def _process_category(self, data):
"""Process category data for FurAffinity."""
subcategory = data.get("subcategory", "gallery")
category_instance, created = CategoryModel.objects.get_or_create(
slug=subcategory
)
if created:
category_instance.name = subcategory.capitalize()
# Process FA-specific categories
if "fa_category" in data:
fa_category = data["fa_category"]
fa_category_instance, _ = CategoryModel.objects.get_or_create(
slug=fa_category.lower().replace(" ", "_")
)
fa_category_instance.name = fa_category
fa_category_instance.save()
category_instance.save()
return category_instance
def _process_post(
self,
data,
source_site_instance,
creator_instance,
category_instance,
file_path_json,
delete,
):
"""Process post data for FurAffinity."""
post_id = str(data.get("id", ""))
post_instance, _ = PostModel.objects.get_or_create(
post_id=post_id, source_site=source_site_instance
)
# Add category
if category_instance:
post_instance.category.add(category_instance)
# Add category to creator
if creator_instance:
creator_instance.refresh_from_db()
creator_instance.categories.add(category_instance)
creator_instance.save()
# Link creator
if creator_instance:
post_instance.creator = creator_instance
# Set creation date
if "date" in data:
post_instance.date_created = timezone.make_aware(
datetime.strptime(data["date"], "%Y-%m-%d %H:%M:%S")
)
# Set mature content flag based on rating
rating = data.get("rating", "").lower()
post_instance.mature = rating in ["mature", "adult"]
# Add description (title + description)
title = data.get("title", "")
description = data.get("description", "")
full_description = f"{title}\n\n{description}" if title else description
if full_description:
self.add_description(
description_text=full_description,
date_str=data["date"],
date_format="%Y-%m-%d %H:%M:%S",
owner_instance=post_instance,
owner_type="post",
file_date=os.path.getmtime(file_path_json),
)
# Add tags
if "tags" in data:
self.add_tags(data["tags"], post_instance)
# Add species as a special tag if present
if "species" in data and data["species"] not in [
"Unspecified / Any",
"Any",
]:
species_tags = [s.strip() for s in data["species"].split("/")]
self.add_tags(species_tags, post_instance)
# Add gender as a special tag if present
if "gender" in data and data["gender"] not in ["Unspecified / Any", "Any"]:
gender_tags = [g.strip() for g in data["gender"].split("/")]
self.add_tags(gender_tags, post_instance)
# Add metadata as JSON field if your model supports it
metadata = {}
for field in ["views", "favorites", "comments", "theme", "fa_category"]:
if field in data:
metadata[field] = data[field]
# If your PostModel has a metadata JSONField, uncomment this
# post_instance.metadata = metadata
# Import the file
file_path = file_path_json.removesuffix(".json")
# Check if the file exists, otherwise try to construct from filename and extension
if not os.path.exists(file_path) and "filename" in data and "extension" in data:
alt_file_path = f"{os.path.dirname(file_path_json)}/{data['filename']}.{data['extension']}"
file_instance = self.import_file(alt_file_path, delete)
else:
file_instance = self.import_file(file_path, delete)
if file_instance:
post_instance.files.add(file_instance)
# Add known image dimensions if available
if not file_instance.width and "width" in data:
file_instance.width = data.get("width")
if not file_instance.height and "height" in data:
file_instance.height = data.get("height")
if "width" in data or "height" in data:
file_instance.save()
post_instance.save()
@ -432,6 +589,7 @@ class Command(BaseCommand):
super().__init__(*args, **kwargs)
self.importers = {
"twitter": TwitterImporter(self),
"furaffinity": FurAffinityImporter(self),
}
# Set up logging