32 string help(
const vector<string>& tokens)
const override;
41 static string initialFilePoolSettingValue()
48 "-types",
"system_rom"),
50 "-types",
"rom disk tape"));
57 controller,
"__filepool",
58 "This is an internal setting. Don't change this directly, " 59 "instead use the 'filepool' command.",
60 initialFilePoolSettingValue())
64 filePoolSetting.
attach(*
this);
73 sha1SumCommand = std::make_unique<Sha1SumCommand>(controller, *
this);
82 filePoolSetting.
detach(*
this);
88 stringBuffer.push_back(filename);
89 pool.emplace(it, sum, time, stringBuffer.back().c_str());
93 void FilePool::remove(Pool::iterator it)
104 bool FilePool::adjust(Pool::iterator it,
const Sha1Sum& newSum)
111 rotate(it, it + 1, newIt);
116 rotate(newIt, it, it + 1);
126 time_t FilePool::PoolEntry::getTime()
128 if (time == time_t(-1)) {
134 void FilePool::PoolEntry::setTime(time_t
t)
140 static bool parse(
char* line,
char* line_end,
141 Sha1Sum& sha1,
const char*& timeStr,
const char*& filename)
143 if ((line_end - line) <= 68)
return false;
146 if (line[40] !=
' ')
return false;
147 if (line[41] !=
' ')
return false;
148 if (line[45] !=
' ')
return false;
149 if (line[49] !=
' ')
return false;
150 if (line[52] !=
' ')
return false;
151 if (line[55] !=
':')
return false;
152 if (line[58] !=
':')
return false;
153 if (line[61] !=
' ')
return false;
154 if (line[66] !=
' ')
return false;
155 if (line[67] !=
' ')
return false;
166 filename = line + 68;
171 void FilePool::readSha1sums()
173 assert(pool.empty());
174 assert(fileMem.
empty());
177 auto size = file.getSize();
180 fileMem[
size] =
'\n';
184 char* data = fileMem.
data();
185 char* data_end = data +
size + 1;
186 while (data != data_end) {
188 char* it =
static_cast<char*
>(memchr(data,
'\n', data_end - data));
189 if (it ==
nullptr) it = data_end;
190 if ((it != data) && (it[-1] ==
'\r')) --it;
195 if (parse(data, it, sum, timeStr, filename)) {
196 pool.emplace_back(sum, timeStr, filename);
200 return !(c ==
'\n' || c ==
'\r');
213 void FilePool::writeSha1sums()
218 if (!file.is_open()) {
221 for (
auto& p : pool) {
222 file << p.sum.toString() <<
" ";
226 assert(p.time != time_t(-1));
229 file <<
" " << p.filename <<
'\n';
237 for (
unsigned i = 0; i < num; ++i) {
239 if (elem ==
"system_rom") {
241 }
else if (elem ==
"rom") {
243 }
else if (elem ==
"disk") {
245 }
else if (elem ==
"tape") {
254 void FilePool::update(
const Setting& setting)
256 assert(&setting == &filePoolSetting); (void)setting;
260 FilePool::Directories FilePool::getDirectories()
const 266 for (
unsigned i = 0; i < numLines; ++i) {
268 bool hasPath =
false;
274 "Expected a list with an even number " 275 "of elements, but got ", line.
getString());
277 for (
unsigned j = 0; j < numItems; j += 2) {
280 if (name ==
"-path") {
283 }
else if (name ==
"-types") {
284 entry.types = parseTypes(interp, value);
291 "Missing -path item: ", line.
getString());
293 if (entry.types == 0) {
295 "Missing -types item: ", line.
getString());
297 result.push_back(entry);
304 File result = getFromPool(sha1sum);
305 if (result.
is_open())
return result;
308 ScanProgress progress;
310 progress.amountScanned = 0;
312 Directories directories;
314 directories = getDirectories();
317 "Error while parsing '__filepool' setting", e.
getMessage());
319 for (
auto& d : directories) {
320 if (d.types & fileType) {
322 result = scanDirectory(sha1sum, path, d.path, progress);
323 if (result.
is_open())
return result;
330 static void reportProgress(
const string& filename,
size_t percentage,
334 "Calculating SHA1 sum for ", filename,
"... ", percentage,
'%');
342 constexpr
size_t STEP_SIZE = 1024 * 1024;
344 auto data = file.
mmap();
348 size_t size = data.size();
350 size_t remaining =
size;
352 bool everShowedProgress =
false;
355 while (remaining > STEP_SIZE) {
356 sha1.
update(&data[done], STEP_SIZE);
358 remaining -= STEP_SIZE;
361 if ((now - lastShowedProgress) > 1000000) {
362 reportProgress(filename, (100 * done) / size, reactor);
363 lastShowedProgress = now;
364 everShowedProgress =
true;
368 sha1.
update(&data[done], remaining);
369 if (everShowedProgress) {
370 reportProgress(filename, 100, reactor);
380 auto last =
distance(begin(pool), e);
382 auto it = begin(pool) + i;
383 if (it->getTime() == time_t(-1)) {
391 File file(it->filename);
393 if (it->time == newTime) {
399 it->setTime(newTime);
401 auto newSum = calcSha1sum(file, reactor);
402 if (newSum == sha1sum) {
409 if (adjust(it, newSum)) {
426 File FilePool::scanDirectory(
427 const Sha1Sum& sha1sum,
const string& directory,
const string& poolPath,
428 ScanProgress& progress)
431 while (dirent* d = dir.
getEntry()) {
438 string file = d->d_name;
439 string path =
strCat(directory,
'/', file);
444 result = scanFile(sha1sum, path, st, poolPath, progress);
446 if ((file !=
".") && (file !=
"..")) {
447 result = scanDirectory(sha1sum, path, poolPath, progress);
450 if (result.
is_open())
return result;
456 File FilePool::scanFile(
const Sha1Sum& sha1sum,
const string& filename,
458 ScanProgress& progress)
460 ++progress.amountScanned;
463 if (now > (progress.lastTime + 250000)) {
464 progress.lastTime = now;
466 "Searching for file with sha1sum ",
467 sha1sum.
toString(),
"...\nIndexing filepool ", poolPath,
468 ": [", progress.amountScanned,
"]: ",
469 std::string_view(filename).substr(poolPath.size()));
476 auto it = findInDatabase(filename);
477 if (it == end(pool)) {
481 auto sum = calcSha1sum(file, reactor);
483 insert(sum, time, filename);
484 if (sum == sha1sum) {
492 assert(filename == it->filename);
493 assert(it->time != time_t(-1));
496 if (it->time == time) {
498 if (it->sum == sha1sum) {
499 return File(filename);
504 auto sum = calcSha1sum(file, reactor);
507 if (sum == sha1sum) {
519 FilePool::Pool::iterator FilePool::findInDatabase(
const string& filename)
527 auto i = pool.size();
530 auto it = begin(pool) + i;
531 if (it->filename == filename) {
533 if (it->getTime() == time_t(-1)) {
548 const auto& filename = file.
getURL();
550 auto it = findInDatabase(filename);
551 assert((it == end(pool)) || (it->time != time_t(-1)));
552 if ((it != end(pool)) && (it->time == time)) {
559 auto sum = calcSha1sum(file, reactor);
560 if (it == end(pool)) {
562 insert(sum, time, filename);
571 int FilePool::signalEvent(
const std::shared_ptr<const Event>& event)
584 :
Command(commandController_,
"sha1sum")
585 , filePool(filePool_)
592 File file(tokens[1].getString());
598 return "Calculate sha1 value for the given file. If the file is " 599 "(g)zipped the sha1 is calculated on the unzipped version.";
std::string getOriginalName()
Get Original filename for this object.
Contains the main loop of openMSX.
bool isRegularFile(const Stat &st)
void update(const uint8_t *data, size_t len)
Incrementally calculate the hash value.
auto sum(InputRange &&range)
const std::string & getMessage() const &
void registerEventListener(EventType type, EventListener &listener, Priority priority=OTHER)
Registers a given object to receive certain events.
auto distance(octet_iterator first, octet_iterator last)
string help(const vector< string > &tokens) const override
Print help for this command.
File getFile(FileType fileType, const Sha1Sum &sha1sum)
Search file with the given sha1sum.
void unregisterEventListener(EventType type, EventListener &listener)
Unregisters a previously registered event listener.
FileContext systemFileContext()
void openofstream(std::ofstream &stream, const std::string &filename)
Open an ofstream in a platform-independent manner.
void checkNumArgs(span< const TclObject > tokens, unsigned exactly, const char *errMessage) const
void printProgress(std::string_view message)
unsigned getListLength(Interpreter &interp) const
TclObject makeTclDict(Args &&... args)
size_t size(std::string_view utf8)
Sha1SumCommand(CommandController &commandController, FilePool &filePool)
EventDistributor & getEventDistributor()
auto upper_bound(ForwardRange &&range, const T &value)
std::string getURL() const
Returns the URL of this file object.
auto find_if(InputRange &&range, UnaryPredicate pred)
static void completeFileName(std::vector< std::string > &tokens, const FileContext &context, const RANGE &extra)
void attach(Observer< T > &observer)
void repaint()
Redraw the display.
FileContext userFileContext(string_view savePath)
This class represents the result of a sha1 calculation (a 160-bit value).
Sha1Sum digest()
Get the final hash.
bool is_sorted(ForwardRange &&range)
auto equal_range(ForwardRange &&range, const T &value)
constexpr const char *const filename
const TclObject & getValue() const final override
Gets the current value of this setting as a TclObject.
void resize(size_t size)
Grow or shrink the memory block.
bool is_open() const
Return true iff this file handle refers to an open file.
const T * data() const
Returns pointer to the start of the memory buffer.
string getUserDataDir()
Get the openMSX data dir in the user's home directory.
time_t getModificationDate()
Get the date/time of last modification.
void sort(RandomAccessRange &&range)
Thanks to enen for testing this on a real cartridge:
TclObject getListIndex(Interpreter &interp, unsigned index) const
std::string toString(time_t time)
Helper class to perform a sha1 calculation.
void printWarning(std::string_view message)
std::string_view getString() const
std::string toString() const
bool isDirectory(const Stat &st)
void parse40(const char *str)
Parse from a 40-character long buffer.
const char *const FILE_CACHE
bool getStat(string_view filename_, Stat &st)
FilePool(CommandController &controller, Reactor &reactor)
void tabCompletion(vector< string > &tokens) const override
Attempt tab completion for this command.
void execute(span< const TclObject > tokens, TclObject &result) override
Execute this command.
Sha1Sum getSha1Sum(File &file)
Calculate sha1sum for the given File object.
time_t fromString(const char *p)
string expandTilde(string_view path)
Simple wrapper around openmdir() / readdir() / closedir() functions.
void detach(Observer< T > &observer)
Interpreter & getInterpreter() const
std::string strCat(Ts &&...ts)
string join(string_view part1, string_view part2)
uint64_t getTime()
Get current (real) time in us.
mat4 rotate(float angle, const vec3 &axis)
span< uint8_t > mmap()
Map file in memory.
struct dirent * getEntry()
Get directory entry for next file.
bool empty() const
No memory allocated?
time_t getModificationDate(const Stat &st)
Get the date/time of last modification.