Differential D12544 Diff 43573 source/blender/editors/asset/intern/asset_indexer.cc

Changeset View

Standalone View

source/blender/editors/asset/intern/asset_indexer.cc

This file was added.

				/*
				* This program is free software; you can redistribute it and/or
				* modify it under the terms of the GNU General Public License
				* as published by the Free Software Foundation; either version 2
				* of the License, or (at your option) any later version.
				*
				* This program is distributed in the hope that it will be useful,
				* but WITHOUT ANY WARRANTY; without even the implied warranty of
				* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
				* GNU General Public License for more details.
				*
				* You should have received a copy of the GNU General Public License
				* along with this program; if not, write to the Free Software Foundation,
				* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
				*/

				/** \file
				* \ingroup edasset
				*/

				#include <fstream>
				#include <iomanip>
				#include <optional>

				#include "ED_asset_indexer.h"

				#include "DNA_asset_types.h"
				#include "DNA_userdef_types.h"

				#include "BLI_fileops.h"
				#include "BLI_hash.hh"
				#include "BLI_linklist.h"
				#include "BLI_path_util.h"
				#include "BLI_serialize.hh"
				#include "BLI_set.hh"
				#include "BLI_string_ref.hh"
				#include "BLI_uuid.h"

				#include "BKE_appdir.h"
				#include "BKE_asset.h"
				#include "BKE_asset_catalog.hh"
				#include "BKE_preferences.h"

				#include "CLG_log.h"

				static CLG_LogRef LOG = {"ed.asset"};

				namespace blender::ed::asset {

				using namespace blender::io::serialize;
				using namespace blender::bke;

				/*
				* The structure of an index file is
				* ```
				* {
				* "version": <file version number>,
				* "entries": [{
				* "name": "<asset name>",
				* "catalog_id": "<catalog_id>",
				* "catalog_name": "<catalog_name>",
				* "description": "<description>",
				* "tags": ["<tag>"]
				* }]
				* }
				* ```
				*
				* `entries`, `description` and `tags` are optional attributes.
				*
				* File browser uses name and idcode separate. We join them together to reduce the fields we
				* need to store inside an index file.
				* File browser also uses group name what is an translatable
				* name. When we store this inside the index an the UI could show the group name in the incorrect
				* language. We solve this by
				*/
				constexpr StringRef ATTRIBUTE_VERSION("version");
				constexpr StringRef ATTRIBUTE_ENTRIES("entries");
				constexpr StringRef ATTRIBUTE_ENTRIES_NAME("name");
				constexpr StringRef ATTRIBUTE_ENTRIES_CATALOG_ID("catalog_id");
				constexpr StringRef ATTRIBUTE_ENTRIES_CATALOG_NAME("catalog_name");
				constexpr StringRef ATTRIBUTE_ENTRIES_DESCRIPTION("description");
				constexpr StringRef ATTRIBUTE_ENTRIES_TAGS("tags");

				/** Abstract class for AssetFile and AssetIndexFile */
				class File {
				public:
				virtual ~File() = default;

				virtual const char *get_file_path() const = 0;

				const bool exists() const
				{
				return BLI_exists(get_file_path());
				}

				const size_t get_file_size() const
				{
				return BLI_file_size(get_file_path());
				}
				};

				/** AssetFile is a source blend file that can be indexed. */
				class AssetFile : public File {
				StringRefNull file_path_;

				public:
				AssetFile(StringRefNull file_path) : file_path_(file_path)
				{
				}

				uint64_t hash() const
				{
				DefaultHash<StringRefNull> hasher;
				return hasher(file_path_);
				}

				std::string get_filename() const
				{
				char filename[FILE_MAX];
				BLI_split_file_part(get_file_path(), filename, sizeof(filename));
				return std::string(filename);
				}

				const char *get_file_path() const override
				{
				return file_path_.c_str();
				}
				};

				/**
				* Helper class to parse a single entry from an AssetIndexFile.
				*/
				struct AssetEntryReader {
				private:
				/**
				* The index files contains multiple entries. A single entry inside the index file is a
				* `blender::io::serialize::ObjectValue`. We keep the lookup table, result of
				* `blender::io::serialize::ObjectValue::create_lookup` locally so we don't need to recreate it
				* every time an attribute inside the entry is requested.
				*/
				ObjectValue::Lookup lookup;

				const StringRefNull get_name_with_idcode() const
				{
				return lookup.lookup(ATTRIBUTE_ENTRIES_NAME)->as_string_value()->value();
				}

				public:
				AssetEntryReader(const ObjectValue &entry) : lookup(entry.create_lookup())
				{
				}

				const ID_Type get_idcode() const
				{
				const StringRefNull name_with_idcode = get_name_with_idcode();
				return GS(name_with_idcode.c_str());
				}

				const StringRef get_name() const
				{
				const StringRefNull name_with_idcode = get_name_with_idcode();
				return name_with_idcode.substr(2);
				}

				const bool has_description() const
				{
				return lookup.contains(ATTRIBUTE_ENTRIES_DESCRIPTION);
				}

				const StringRefNull get_description() const
				{
				return lookup.lookup(ATTRIBUTE_ENTRIES_DESCRIPTION)->as_string_value()->value();
				}

				const StringRefNull get_catalog_name() const
				{
				return lookup.lookup(ATTRIBUTE_ENTRIES_CATALOG_NAME)->as_string_value()->value();
				}

				const CatalogID get_catalog_id() const
				{
				const std::string &catalog_id =
				lookup.lookup(ATTRIBUTE_ENTRIES_CATALOG_ID)->as_string_value()->value();
				CatalogID catalog_uuid(catalog_id);
				return catalog_uuid;
				}

				void add_tags_to_meta_data(AssetMetaData *asset_data) const
				{
				const ObjectValue::LookupValue *value_ptr = lookup.lookup_ptr(ATTRIBUTE_ENTRIES_TAGS);
				if (value_ptr == nullptr) {
				return;
				}

				const ArrayValue array_value = (value_ptr)->as_array_value();
				const ArrayValue::Items &elements = array_value->elements();
				for (const ArrayValue::Item &item : elements) {
				const StringRefNull tag_name = item->as_string_value()->value();
				BKE_asset_metadata_tag_add(asset_data, tag_name.c_str());
				}
				}
				};

				struct AssetEntryWriter {
				private:
				ObjectValue::Items &attributes;

				public:
				AssetEntryWriter(ObjectValue &entry) : attributes(entry.elements())
				{
				}

				/**
				* Adds an idcode and name to the attributes to store inside the index.
				* They are encoded together to save some space inside the index.
				*/
				void add_id_name(const short idcode, const StringRefNull name)
				{
				char idcode_prefix[2];
				/* Similar to `BKE_libblock_alloc`. */
				((short )idcode_prefix) = idcode;
				std::string name_with_idcode = std::string(idcode_prefix) + name;

				attributes.append_as(std::pair(ATTRIBUTE_ENTRIES_NAME, new StringValue(name_with_idcode)));
				}

				void add_catalog_id(const CatalogID &catalog_id)
				{
				char catalog_id_str[UUID_STRING_LEN];
				BLI_uuid_format(catalog_id_str, catalog_id);
				attributes.append_as(std::pair(ATTRIBUTE_ENTRIES_CATALOG_ID, new StringValue(catalog_id_str)));
				}

				void add_catalog_name(const StringRefNull catalog_name)
				{
				attributes.append_as(std::pair(ATTRIBUTE_ENTRIES_CATALOG_NAME, new StringValue(catalog_name)));
				}

				void add_description(const StringRefNull description)
				{
				attributes.append_as(std::pair(ATTRIBUTE_ENTRIES_DESCRIPTION, new StringValue(description)));
				}

				void add_tags(const ListBase /* AssetTag / asset_tags)
				{
				ArrayValue *tags = new ArrayValue();
				attributes.append_as(std::pair(ATTRIBUTE_ENTRIES_TAGS, tags));
				ArrayValue::Items &tag_items = tags->elements();

				LISTBASE_FOREACH (AssetTag *, tag, asset_tags) {
				tag_items.append_as(new StringValue(tag->name));
				}
				}
				};

				static void init_value_from_file_indexer_entry(AssetEntryWriter &result,
				const FileIndexerEntry *indexer_entry)
				{
				const BLODataBlockInfo &datablock_info = indexer_entry->datablock_info;

				result.add_id_name(indexer_entry->idcode, datablock_info.name);

				const AssetMetaData &asset_data = *datablock_info.asset_data;
				result.add_catalog_id(asset_data.catalog_id);
				result.add_catalog_name(asset_data.catalog_simple_name);

				if (asset_data.description != nullptr) {
				result.add_description(asset_data.description);
				}

				if (!BLI_listbase_is_empty(&asset_data.tags)) {
				result.add_tags(&asset_data.tags);
				}

				/* TODO: asset_data.IDProperties */
				}

				static void init_value_from_file_indexer_entries(ObjectValue &result,
				const FileIndexerEntries &indexer_entries)
				{
				ArrayValue *entries = new ArrayValue();
				ArrayValue::Items &items = entries->elements();

				for (LinkNode *ln = indexer_entries.entries; ln; ln = ln->next) {
				const FileIndexerEntry indexer_entry = static_cast<const FileIndexerEntry >(ln->link);
				/* We also get non asset types (Brushes/Workspaces), when browsing using the asset browser. */
				if (indexer_entry->datablock_info.asset_data == nullptr) {
				continue;
				}
				ObjectValue *entry_value = new ObjectValue();
				AssetEntryWriter entry(*entry_value);
				init_value_from_file_indexer_entry(entry, indexer_entry);
				items.append_as(entry_value);
				}

				/* When no entries to index, we should not store the entries attribute as this would make the
				* size bigger than the MIN_FILE_SIZE_WITH_ENTRIES. */
				if (items.is_empty()) {
				delete entries;
				return;
				}

				ObjectValue::Items &attributes = result.elements();
				attributes.append_as(std::pair(ATTRIBUTE_ENTRIES, entries));
				}

				static void init_indexer_entry_from_value(FileIndexerEntry &indexer_entry,
				const AssetEntryReader &entry)
				{
				indexer_entry.idcode = entry.get_idcode();

				const std::string &name = entry.get_name();
				BLI_strncpy(
				indexer_entry.datablock_info.name, name.c_str(), sizeof(indexer_entry.datablock_info.name));

				AssetMetaData *asset_data = BKE_asset_metadata_create();
				indexer_entry.datablock_info.asset_data = asset_data;

				if (entry.has_description()) {
				const std::string &description = entry.get_description();
				char description_c_str = static_cast<char >(MEM_mallocN(description.length() + 1, __func__));
				BLI_strncpy(description_c_str, description.c_str(), description.length() + 1);
				asset_data->description = description_c_str;
				}

				const std::string &catalog_name = entry.get_catalog_name();
				BLI_strncpy(asset_data->catalog_simple_name,
				catalog_name.c_str(),
				sizeof(asset_data->catalog_simple_name));

				asset_data->catalog_id = entry.get_catalog_id();

				entry.add_tags_to_meta_data(asset_data);
				}

				static int init_indexer_entries_from_value(FileIndexerEntries &indexer_entries,
				const ObjectValue &value)
				{
				const ObjectValue::Lookup attributes = value.create_lookup();
				const ObjectValue::LookupValue *entries_value = attributes.lookup_ptr(ATTRIBUTE_ENTRIES);
				BLI_assert(entries_value != nullptr);

				if (entries_value == nullptr) {
				return 0;
				}

				int num_entries_read = 0;
				const ArrayValue::Items elements = (*entries_value)->as_array_value()->elements();
				for (ArrayValue::Item element : elements) {
				const AssetEntryReader asset_entry(*element->as_object_value());

				FileIndexerEntry entry = static_cast<FileIndexerEntry >(
				MEM_callocN(sizeof(FileIndexerEntry), __func__));
				init_indexer_entry_from_value(*entry, asset_entry);

				BLI_linklist_prepend(&indexer_entries.entries, entry);
				num_entries_read += 1;
				}

				return num_entries_read;
				}

				/**
				* Struct referencing the directory containing all indices of a asset library.
				*
				* The AssetLibraryIndex instance is used to keep track of unused file indices. During reading any
				* used indices are removed from the list and at the end when reading is finished the unused
				* indices are removed.
				*/
				struct AssetLibraryIndex {
				bUserAssetLibrary *library;

				/**
				* Tracks indices that haven been used yet.
				*
				* Contains absolute paths to the indices.
				*/
				Set<std::string> unused_file_indices;

				/**
				* Folder where the indices of `library` are stored.
				*
				* Must including trailing directory separator.
				*/
				std::string indices_base_path;

				public:
				AssetLibraryIndex(const AssetLibraryReference *library_ref)
				{
				library = BKE_preferences_asset_library_find_from_index(&U, library_ref->custom_library_index);
				init_indices_base_path();
				}

				const StringRefNull get_name() const
				{
				return StringRefNull(library->name);
				}

				uint64_t hash() const
				{
				DefaultHash<StringRefNull> hasher;
				return hasher(get_library_file_path());
				}

				const StringRefNull get_library_file_path() const
				{
				return StringRefNull(library->path);
				}

				/**
				* Initializes the `indices_base_path` attribute.
				*
				* `BKE_appdir_folder_caches/asset-library-indices/<asset-library-name-hash>_<asset-library-name>/`
				*/
				void init_indices_base_path()
				{
				char index_path[FILE_MAX];
				BKE_appdir_folder_caches(index_path, sizeof(index_path));

				BLI_path_append(index_path, sizeof(index_path), "asset-library-indices");

				std::stringstream ss;
				ss << std::setfill('0') << std::setw(16) << std::hex << hash() << "_" << get_name() << "/";
				BLI_path_append(index_path, sizeof(index_path), ss.str().c_str());

				indices_base_path = std::string(index_path);
				}

				/**
				* Returns the full path to the index file of the given `asset_file`.
				*
				* `{indices_base_path}/{asset-file_hash}_{asset-file-filename}.index.json`.
				*/
				std::string index_file_path(const AssetFile &asset_file) const
				{
				std::stringstream ss;
				ss << indices_base_path;
				ss << std::setfill('0') << std::setw(16) << std::hex << asset_file.hash() << "_"
				<< asset_file.get_filename() << ".index.json";
				return ss.str();
				}

				/**
				* Initialize to keep track of unused file indices.
				*/
				void init_unused_index_files()
				{
				const char *index_path = indices_base_path.c_str();
				if (!BLI_is_dir(index_path)) {
				return;
				}
				struct direntry *dir_entries = nullptr;
				int num_entries = BLI_filelist_dir_contents(index_path, &dir_entries);
				for (int i = 0; i < num_entries; i++) {
				struct direntry *entry = &dir_entries[i];
				if (BLI_str_endswith(entry->relname, ".index.json")) {
				unused_file_indices.add_as(std::string(entry->path));
				}
				}

				BLI_filelist_free(dir_entries, num_entries);
				}

				void mark_as_used(const std::string &filename)
				{
				unused_file_indices.remove(filename);
				}

				int remove_unused_index_files() const
				{
				int num_files_deleted = 0;
				for (const std::string &unused_index : unused_file_indices) {
				const char *file_path = unused_index.c_str();
				CLOG_INFO(&LOG, 2, "Remove unused index file [%s].", file_path);
				BLI_delete(file_path, false, false);
				num_files_deleted++;
				}

				return num_files_deleted;
				}
				};

				/**
				* Instance of this class represents the contents of an asset index file.
				*
				* ```
				* {
				* "version": {version},
				* "entries": ...
				* }
				* ```
				*/
				struct AssetIndex {
				/**
				* Versions are written to each index file. When reading the version is checked against
				* `CURRENT_VERSION` to make sure we can use the index. Developer should increase
				* `CURRENT_VERSION` when changes are made to the structure of the stored index.
				*/
				static const int CURRENT_VERSION = 1;

				/**
				* Version number to use when version couldn't be read from an index file.
				*/
				const int UNKNOWN_VERSION = -1;

				/**
				* `blender::io::serialize::Value` represeting the contents of an index file.
				*
				* Value is used over ObjectValue as the contents of the index could be corrupted and doesn't
				* represent an object. In case a corrupted file is detected the `get_version` would return
				* UNKNOWN_VERSION.`
				*/
				Value *contents;

				/**
				* Constructor for when creating/updating an asset index file.
				* `contents` are filled from the given `indexer_entries`.
				*/
				AssetIndex(const FileIndexerEntries &indexer_entries)
				{
				ObjectValue *root = new ObjectValue();
				contents = root;
				ObjectValue::Items &root_attributes = root->elements();
				root_attributes.append_as(std::pair(ATTRIBUTE_VERSION, new IntValue(CURRENT_VERSION)));

				init_value_from_file_indexer_entries(*root, indexer_entries);
				}

				/**
				* Constructor when reading an asset index file.
				* `contents` are read from the given `value`.
				*/
				AssetIndex(Value *value) : contents(value)
				{
				}

				virtual ~AssetIndex()
				{
				delete contents;
				}

				const int get_version() const
				{
				const ObjectValue *root = contents->as_object_value();
				if (root == nullptr) {
				return UNKNOWN_VERSION;
				}
				const ObjectValue::Lookup attributes = root->create_lookup();
				const ObjectValue::LookupValue *version_value = attributes.lookup_ptr(ATTRIBUTE_VERSION);
				if (version_value == nullptr) {
				return UNKNOWN_VERSION;
				}
				return (*version_value)->as_int_value()->value();
				}

				const bool is_latest_version() const
				{
				return get_version() == CURRENT_VERSION;
				}

				/**
				* Extract the contents of this index into the given FileIndexerEntries.
				*/
				const int extract_into(FileIndexerEntries &indexer_entries) const
				{
				const ObjectValue *root = contents->as_object_value();
				const int num_entries_read = init_indexer_entries_from_value(indexer_entries, *root);
				return num_entries_read;
				}
				};

				class AssetIndexFile : public File {
				public:
				AssetLibraryIndex &library_index;
				/**
				* Asset index files with a size smaller than this attribute would be considered to not contain
				* any entries.
				*/
				const size_t MIN_FILE_SIZE_WITH_ENTRIES = 32;
				std::string filename;

				AssetIndexFile(AssetLibraryIndex &library_index, AssetFile &asset_filename)
				: library_index(library_index), filename(library_index.index_file_path(asset_filename))
				{
				}

				void mark_as_used()
				{
				library_index.mark_as_used(filename);
				}

				const char *get_file_path() const override
				{
				return filename.c_str();
				}

				/**
				* Returns whether the index file is older than the given asset file.
				*/
				const bool is_older_than(AssetFile &asset_file) const
				{
				return BLI_file_older(get_file_path(), asset_file.get_file_path());
				}

				/**
				* Check whether the index file contains entries without opening the file.
				*/
				const bool constains_entries() const
				{
				const size_t file_size = get_file_size();
				return file_size >= MIN_FILE_SIZE_WITH_ENTRIES;
				}

				std::unique_ptr<AssetIndex> read_contents() const
				{
				JsonFormatter formatter;
				std::ifstream is;
				is.open(filename);
				std::unique_ptr<Value> read_data = formatter.deserialize(is);
				is.close();

				return std::make_unique<AssetIndex>(read_data.release());
				}

				bool ensure_parent_path_exists() const
				{
				/* `BLI_make_existing_file` only ensures parent path, otherwise than expected from the name of
				* the function. */
				return BLI_make_existing_file(get_file_path());
				}

				void write_contents(AssetIndex &content)
				{
				JsonFormatter formatter;
				if (!ensure_parent_path_exists()) {
				CLOG_ERROR(&LOG, "Index not created: couldn't create folder [%s].", get_file_path());
				return;
				}

				std::ofstream os;
				os.open(filename, std::ios::out \| std::ios::trunc);
				formatter.serialize(os, *content.contents);
				os.close();
				}
				};

				static eFileIndexerResult read_index(const char *filename,
				FileIndexerEntries *entries,
				int *r_read_entries_len,
				void *user_data)
				{
				AssetLibraryIndex &library_index = static_cast<AssetLibraryIndex >(user_data);
				AssetFile asset_file(filename);
				AssetIndexFile asset_index_file(library_index, asset_file);

				if (!asset_index_file.exists()) {
				return FILE_INDEXER_NEEDS_UPDATE;
				}

				/* Mark index as used, even when it will be recreated. When not done it would remove the index
				* when the indexing has finished (see `AssetLibraryIndex.remove_unused_index_files`), thereby
				* removing the newly created index.
				*/
				asset_index_file.mark_as_used();

				if (asset_index_file.is_older_than(asset_file)) {
				CLOG_INFO(
				&LOG,
				3,
				"Asset index file [%s] needs to be refreshed as it is older than the asset file [%s].",
				asset_index_file.filename.c_str(),
				filename);
				return FILE_INDEXER_NEEDS_UPDATE;
				}

				if (!asset_index_file.constains_entries()) {
				CLOG_INFO(&LOG,
				3,
				"Asset file index is to small to contain any entries. [%s]",
				asset_index_file.filename.c_str());
				*r_read_entries_len = 0;
				return FILE_INDEXER_ENTRIES_LOADED;
				}

				std::unique_ptr<AssetIndex> contents = asset_index_file.read_contents();
				if (!contents->is_latest_version()) {
				CLOG_INFO(&LOG,
				3,
				"Asset file index is ignoredi; expected version %d but file is version %d [%s].",
				AssetIndex::CURRENT_VERSION,
				contents->get_version(),
				asset_index_file.filename.c_str());
				return FILE_INDEXER_NEEDS_UPDATE;
				}

				const int read_entries_len = contents->extract_into(*entries);
				CLOG_INFO(&LOG, 1, "Read %d entries from asset index for [%s].", read_entries_len, filename);
				*r_read_entries_len = read_entries_len;

				return FILE_INDEXER_ENTRIES_LOADED;
				}

				static void update_index(const char filename, FileIndexerEntries entries, void *user_data)
				{
				AssetLibraryIndex &library_index = static_cast<AssetLibraryIndex >(user_data);
				AssetFile asset_file(filename);
				AssetIndexFile asset_index_file(library_index, asset_file);
				CLOG_INFO(&LOG,
				1,
				"Update asset index for [%s] store index in [%s].",
				asset_file.get_file_path(),
				asset_index_file.get_file_path());

				AssetIndex content(*entries);
				asset_index_file.write_contents(content);
				}

				static void init_user_data(const AssetLibraryReference library_reference)
				{
				AssetLibraryIndex *library_index = new AssetLibraryIndex(library_reference);
				library_index->init_unused_index_files();
				return library_index;
				}

				static void free_user_data(void *user_data)
				{
				AssetLibraryIndex index = static_cast<AssetLibraryIndex >(user_data);
				delete index;
				}

				static void filelist_finished(void *user_data)
				{
				AssetLibraryIndex &library_index = static_cast<AssetLibraryIndex >(user_data);
				const int num_indices_removed = library_index.remove_unused_index_files();
				if (num_indices_removed > 0) {
				CLOG_INFO(&LOG, 1, "Removed %d unused indices.", num_indices_removed);
				}
				}

				constexpr FileIndexer asset_indexer()
				{
				FileIndexer indexer = {nullptr};
				indexer.read_index = read_index;
				indexer.update_index = update_index;
				indexer.init_user_data = init_user_data;
				indexer.free_user_data = free_user_data;
				indexer.filelist_finished = filelist_finished;
				return indexer;
				}

				} // namespace blender::ed::asset

				extern "C" {
				const FileIndexer file_indexer_asset = blender::ed::asset::asset_indexer();
				}