diff --git a/backend/apps/archive/management/commands/import_data.py b/backend/apps/archive/management/commands/import_data.py index dc52454..8d4c988 100644 --- a/backend/apps/archive/management/commands/import_data.py +++ b/backend/apps/archive/management/commands/import_data.py @@ -395,31 +395,188 @@ class TwitterImporter(BaseImporter): post_instance.save() + +class FurAffinityImporter(BaseImporter): + """Importer for FurAffinity data.""" + + def import_data( + self, data: Dict[str, Any], file_path_json: str, delete: bool + ) -> None: + """Import FurAffinity data from JSON into the database.""" + try: + category = data.get("category", "furaffinity") + source_site_instance = self.get_or_create_source_site(category) + + # Process creator + creator_instance = self._process_creator(data, source_site_instance) + + # Process category + category_instance = self._process_category(data) + + # Process post + self._process_post( + data, + source_site_instance, + creator_instance, + category_instance, + file_path_json, + delete, ) + except Exception as e: + self.log_error(f"Error importing FurAffinity data: {str(e)}") + def _process_creator(self, data, source_site_instance): + """Process creator data for FurAffinity.""" + artist = data.get("artist", "") + artist_url = data.get("artist_url", artist.lower()) + creator_instance, _ = CreatorModel.objects.get_or_create( + slug=artist_url, source_site=source_site_instance + ) + + creator_instance.name = artist + creator_instance.creator_id = artist_url + + # We don't have creator creation date in FurAffinity data + # Using post date as an approximation + if "date" in data: + creator_instance.date_created = timezone.make_aware( + datetime.strptime(data["date"], "%Y-%m-%d %H:%M:%S") ) creator_instance.save() + return creator_instance + + def _process_category(self, data): + """Process category data for FurAffinity.""" + subcategory = data.get("subcategory", "gallery") + + category_instance, created = CategoryModel.objects.get_or_create( + slug=subcategory ) + if created: + category_instance.name = subcategory.capitalize() + # Process FA-specific categories + if "fa_category" in data: + fa_category = data["fa_category"] + fa_category_instance, _ = CategoryModel.objects.get_or_create( + slug=fa_category.lower().replace(" ", "_") + ) + fa_category_instance.name = fa_category + fa_category_instance.save() + category_instance.save() + return category_instance + def _process_post( + self, + data, + source_site_instance, + creator_instance, + category_instance, + file_path_json, + delete, + ): + """Process post data for FurAffinity.""" + post_id = str(data.get("id", "")) + post_instance, _ = PostModel.objects.get_or_create( + post_id=post_id, source_site=source_site_instance + ) + # Add category + if category_instance: + post_instance.category.add(category_instance) + + # Add category to creator + if creator_instance: + creator_instance.refresh_from_db() + creator_instance.categories.add(category_instance) + creator_instance.save() + + # Link creator + if creator_instance: + post_instance.creator = creator_instance + + # Set creation date + if "date" in data: + post_instance.date_created = timezone.make_aware( + datetime.strptime(data["date"], "%Y-%m-%d %H:%M:%S") + ) + + # Set mature content flag based on rating + rating = data.get("rating", "").lower() + post_instance.mature = rating in ["mature", "adult"] + + # Add description (title + description) + title = data.get("title", "") + description = data.get("description", "") + + full_description = f"{title}\n\n{description}" if title else description + + if full_description: + self.add_description( + description_text=full_description, + date_str=data["date"], + date_format="%Y-%m-%d %H:%M:%S", + owner_instance=post_instance, + owner_type="post", + file_date=os.path.getmtime(file_path_json), + ) + + # Add tags + if "tags" in data: + self.add_tags(data["tags"], post_instance) + + # Add species as a special tag if present + if "species" in data and data["species"] not in [ + "Unspecified / Any", + "Any", + ]: + species_tags = [s.strip() for s in data["species"].split("/")] + self.add_tags(species_tags, post_instance) + + # Add gender as a special tag if present + if "gender" in data and data["gender"] not in ["Unspecified / Any", "Any"]: + gender_tags = [g.strip() for g in data["gender"].split("/")] + self.add_tags(gender_tags, post_instance) + + # Add metadata as JSON field if your model supports it + metadata = {} + + for field in ["views", "favorites", "comments", "theme", "fa_category"]: + if field in data: + metadata[field] = data[field] + + # If your PostModel has a metadata JSONField, uncomment this + # post_instance.metadata = metadata + + # Import the file file_path = file_path_json.removesuffix(".json") + # Check if the file exists, otherwise try to construct from filename and extension + if not os.path.exists(file_path) and "filename" in data and "extension" in data: + alt_file_path = f"{os.path.dirname(file_path_json)}/{data['filename']}.{data['extension']}" + file_instance = self.import_file(alt_file_path, delete) + else: + file_instance = self.import_file(file_path, delete) if file_instance: post_instance.files.add(file_instance) + # Add known image dimensions if available + if not file_instance.width and "width" in data: + file_instance.width = data.get("width") if not file_instance.height and "height" in data: file_instance.height = data.get("height") if "width" in data or "height" in data: + file_instance.save() + post_instance.save() @@ -432,6 +589,7 @@ class Command(BaseCommand): super().__init__(*args, **kwargs) self.importers = { "twitter": TwitterImporter(self), + "furaffinity": FurAffinityImporter(self), } # Set up logging