openMSX
FilePoolCore.hh
Go to the documentation of this file.
1 #ifndef FILEPOOLCORE_HH
2 #define FILEPOOLCORE_HH
3 
4 #include "FileOperations.hh"
5 #include "ObjectPool.hh"
6 #include "MemBuffer.hh"
7 #include "SimpleHashSet.hh"
8 #include "sha1.hh"
9 #include "xxhash.hh"
10 #include <cassert>
11 #include <cstdint>
12 #include <ctime>
13 #include <functional>
14 #include <string>
15 #include <string_view>
16 #include <vector>
17 
18 namespace openmsx {
19 
20 class File;
21 
22 enum class FileType {
23  NONE = 0,
24  SYSTEM_ROM = 1, ROM = 2, DISK = 4, TAPE = 8
25 };
27  return static_cast<FileType>(int(x) | int(y));
28 }
30  return static_cast<FileType>(int(x) & int(y));
31 }
33  x = x | y;
34  return x;
35 }
36 
38 {
39 public:
40  struct Dir {
41  std::string path;
43  };
44  using Directories = std::vector<Dir>;
45 
46 public:
47  FilePoolCore(std::string filecache,
48  std::function<Directories()> getDirectories,
49  std::function<void(const std::string&)> reportProgress);
50  ~FilePoolCore();
51 
56  [[nodiscard]] File getFile(FileType fileType, const Sha1Sum& sha1sum);
57 
62  [[nodiscard]] Sha1Sum getSha1Sum(File& file);
63 
68  void abort() { stop = true; }
69 
70 private:
71  struct ScanProgress {
72  uint64_t lastTime;
73  unsigned amountScanned;
74  };
75 
76  struct Entry {
77  Entry(const Sha1Sum& s, time_t t, std::string_view f)
78  : filename(f), time(t), sum(s)
79  {
80  assert(time != Date::INVALID_TIME_T);
81  }
82  Entry(const Sha1Sum& s, const char* t, std::string_view f)
83  : filename(f), timeStr(t), sum(s)
84  {
85  assert(timeStr != nullptr);
86  }
87 
88  time_t getTime();
89  void setTime(time_t t);
90 
91  // - At least one of 'timeStr' or 'time' is valid.
92  // - 'filename' and 'timeStr' are non-owning pointers.
93  std::string_view filename;
94  const char* timeStr = nullptr; // might be nullptr
95  time_t time = Date::INVALID_TIME_T;
96  Sha1Sum sum;
97  };
98 
99  using Pool = ObjectPool<Entry>;
100  using Index = Pool::Index;
101  using Sha1Index = std::vector<Index>; // sorted on sha1sum
102 
103  class FilenameIndexHelper {
104  public:
105  FilenameIndexHelper(const Pool& p) : pool(p) {}
106  std::string_view get(std::string_view s) const { return s; }
107  std::string_view get(Index idx) const { return pool[idx].filename; }
108  private:
109  const Pool& pool;
110  };
111  struct FilenameIndexHash : FilenameIndexHelper {
112  FilenameIndexHash(const Pool& p) : FilenameIndexHelper(p) {}
113  template<typename T> auto operator()(T t) const {
114  XXHasher hasher;
115  return hasher(get(t));
116  }
117  };
118  struct FilenameIndexEqual : FilenameIndexHelper {
119  FilenameIndexEqual(const Pool& p) : FilenameIndexHelper(p) {}
120  template<typename T1, typename T2> bool operator()(T1 x, T2 y) const {
121  return get(x) == get(y);
122  }
123  };
124  // Hash indexed by filename, points to a full object in 'pool'
125  using FilenameIndex = SimpleHashSet<Index, Index(-1), FilenameIndexHash, FilenameIndexEqual>;
126 
127 private:
128  void insert(const Sha1Sum& sum, time_t time, const std::string& filename);
129  Sha1Index::iterator getSha1Iterator(Index idx, Entry& entry);
130  void remove(Sha1Index::iterator it);
131  void remove(Index idx);
132  void remove(Index idx, Entry& entry);
133  bool adjustSha1(Sha1Index::iterator it, Entry& entry, const Sha1Sum& newSum);
134  bool adjustSha1(Index idx, Entry& entry, const Sha1Sum& newSum);
135 
136  void readSha1sums();
137  void writeSha1sums();
138 
139  File getFromPool(const Sha1Sum& sha1sum);
140  File scanDirectory(const Sha1Sum& sha1sum,
141  const std::string& directory,
142  const std::string& poolPath,
143  ScanProgress& progress);
144  File scanFile(const Sha1Sum& sha1sum,
145  const std::string& filename,
146  const FileOperations::Stat& st,
147  const std::string& poolPath,
148  ScanProgress& progress);
149  Sha1Sum calcSha1sum(File& file);
150  std::pair<Index, Entry*> findInDatabase(std::string_view filename);
151 
152 private:
153  std::string filecache; // path of the '.filecache' file.
154  std::function<Directories()> getDirectories;
155  std::function<void(const std::string&)> reportProgress;
156 
157  MemBuffer<char> fileMem; // content of initial .filecache
158  std::vector<std::string> stringBuffer; // owns strings that are not in 'fileMem'
159 
160  Pool pool; // the actual entries
161  Sha1Index sha1Index; // entries accessible via sha1, sorted on 'CompareSha1'
162  FilenameIndex filenameIndex{FilenameIndexHash(pool), FilenameIndexEqual(pool)}; // accessible via filename
163 
164  bool stop = false; // abort long search (set via reportProgress callback)
165  bool needWrite = false; // dirty '.filecache'? write on exit
166 
167  friend class CompareSha1;
168 };
169 
170 } // namespace openmsx
171 
172 #endif
openmsx::FilePoolCore::Dir
Definition: FilePoolCore.hh:40
openmsx::FilePoolCore
Definition: FilePoolCore.hh:38
xxhash.hh
SimpleHashSet
Definition: SimpleHashSet.hh:46
openmsx::FileType
FileType
Definition: FilePoolCore.hh:22
openmsx::FilePoolCore::Dir::path
std::string path
Definition: FilePoolCore.hh:41
openmsx::FilePoolCore::Directories
std::vector< Dir > Directories
Definition: FilePoolCore.hh:44
openmsx::FileOperations::Stat
struct stat Stat
Definition: FileOperations.hh:218
sha1.hh
ObjectPool< Entry >::Index
uint32_t Index
Definition: ObjectPool.hh:54
openmsx::FilePoolCore::Dir::types
FileType types
Definition: FilePoolCore.hh:42
t
TclObject t
Definition: TclObject_test.cc:264
XXHasher
Definition: xxhash.hh:153
openmsx::Date::INVALID_TIME_T
constexpr time_t INVALID_TIME_T
Definition: Date.hh:10
openmsx::Timer::getTime
uint64_t getTime()
Get current (real) time in us.
Definition: Timer.cc:7
openmsx::FilePoolCore::getSha1Sum
Sha1Sum getSha1Sum(File &file)
Calculate sha1sum for the given File object.
Definition: FilePoolCore.cc:454
openmsx::CompareSha1
Definition: FilePoolCore.cc:17
openmsx::operator&
FileType operator&(FileType x, FileType y)
Definition: FilePoolCore.hh:29
openmsx::filename
constexpr const char *const filename
Definition: FirmwareSwitch.cc:10
ObjectPool< Entry >
openmsx::FilePoolCore::getFile
File getFile(FileType fileType, const Sha1Sum &sha1sum)
Search file with the given sha1sum.
Definition: FilePoolCore.cc:247
openmsx::Sha1Sum
This class represents the result of a sha1 calculation (a 160-bit value).
Definition: sha1.hh:20
MemBuffer.hh
openmsx::FileType::ROM
@ ROM
openmsx::FileType::NONE
@ NONE
FileOperations.hh
ObjectPool.hh
openmsx::x
constexpr KeyMatrixPosition x
Keyboard bindings.
Definition: Keyboard.cc:1419
SimpleHashSet.hh
openmsx::File
Definition: File.hh:16
openmsx::FilePoolCore::FilePoolCore
FilePoolCore(std::string filecache, std::function< Directories()> getDirectories, std::function< void(const std::string &)> reportProgress)
Definition: FilePoolCore.cc:31
openmsx::operator|
FileType operator|(FileType x, FileType y)
Definition: FilePoolCore.hh:26
openmsx::FilePoolCore::abort
void abort()
This is only meaningful to call from within the 'reportProgress' callback (constructor parameter).
Definition: FilePoolCore.hh:68
openmsx
This file implemented 3 utility functions:
Definition: Autofire.cc:5
openmsx::operator|=
FileType & operator|=(FileType &x, FileType y)
Definition: FilePoolCore.hh:32
openmsx::FilePoolCore::~FilePoolCore
~FilePoolCore()
Definition: FilePoolCore.cc:45
sum
auto sum(InputRange &&range)
Definition: stl.hh:293