openMSX
serialize.cc
Go to the documentation of this file.
1 #include "serialize.hh"
2 #include "Base64.hh"
3 #include "HexDump.hh"
4 #include "XMLLoader.hh"
5 #include "XMLElement.hh"
6 #include "ConfigException.hh"
7 #include "XMLException.hh"
8 #include "DeltaBlock.hh"
9 #include "MemBuffer.hh"
10 #include "FileOperations.hh"
11 #include "Version.hh"
12 #include "Date.hh"
13 #include "cstdiop.hh" // for dup()
14 #include <cstring>
15 #include <limits>
16 
17 using std::string;
18 
19 namespace openmsx {
20 
21 template<typename Derived>
22 void ArchiveBase<Derived>::attribute(const char* name, const char* value)
23 {
24  string valueStr(value);
25  self().attribute(name, valueStr);
26 }
27 template class ArchiveBase<MemOutputArchive>;
28 template class ArchiveBase<XmlOutputArchive>;
29 
31 
33  : lastId(0)
34 {
35 }
36 
37 unsigned OutputArchiveBase2::generateID1(const void* p)
38 {
39  #ifdef linux
40  assert("Can't serialize ID of object located on the stack" &&
41  !addressOnStack(p));
42  #endif
43  ++lastId;
44  assert(polyIdMap.find(p) == end(polyIdMap));
45  polyIdMap[p] = lastId;
46  return lastId;
47 }
48 unsigned OutputArchiveBase2::generateID2(
49  const void* p, const std::type_info& typeInfo)
50 {
51  #ifdef linux
52  assert("Can't serialize ID of object located on the stack" &&
53  !addressOnStack(p));
54  #endif
55  ++lastId;
56  auto key = std::make_pair(p, std::type_index(typeInfo));
57  assert(idMap.find(key) == end(idMap));
58  idMap[key] = lastId;
59  return lastId;
60 }
61 
62 unsigned OutputArchiveBase2::getID1(const void* p)
63 {
64  auto it = polyIdMap.find(p);
65  return it != end(polyIdMap) ? it->second : 0;
66 }
67 unsigned OutputArchiveBase2::getID2(
68  const void* p, const std::type_info& typeInfo)
69 {
70  auto it = idMap.find({p, std::type_index(typeInfo)});
71  return it != end(idMap) ? it->second : 0;
72 }
73 
74 
75 template<typename Derived>
77  const char* tag, const void* data_, size_t len, bool /*diff*/)
78 {
79  auto* data = static_cast<const uint8_t*>(data_);
80 
81  string encoding;
82  string tmp;
83  if (false) {
84  // useful for debugging
85  encoding = "hex";
86  tmp = HexDump::encode(data, len);
87  } else if (false) {
88  encoding = "base64";
89  tmp = Base64::encode(data, len);
90  } else {
91  encoding = "gz-base64";
92  // TODO check for overflow?
93  auto dstLen = uLongf(len + len / 1000 + 12 + 1); // worst-case
94  MemBuffer<byte> buf(dstLen);
95  if (compress2(buf.data(), &dstLen,
96  reinterpret_cast<const Bytef*>(data),
97  uLong(len), 9)
98  != Z_OK) {
99  throw MSXException("Error while compressing blob.");
100  }
101  tmp = Base64::encode(buf.data(), dstLen);
102  }
103  this->self().beginTag(tag);
104  this->self().attribute("encoding", encoding);
105  Saver<string> saver;
106  saver(this->self(), tmp, false);
107  this->self().endTag(tag);
108 }
109 
112 
114 
116 {
117  auto it = idMap.find(id);
118  return it != end(idMap) ? it->second : nullptr;
119 }
120 
121 void InputArchiveBase2::addPointer(unsigned id, const void* p)
122 {
123  assert(idMap.find(id) == end(idMap));
124  idMap[id] = const_cast<void*>(p);
125 }
126 
127 unsigned InputArchiveBase2::getId(const void* ptr) const
128 {
129  for (const auto& p : idMap) {
130  if (p.second == ptr) return p.first;
131  }
132  return 0;
133 }
134 
135 template<typename Derived>
137  const char* tag, void* data, size_t len, bool /*diff*/)
138 {
139  this->self().beginTag(tag);
140  string encoding;
141  this->self().attribute("encoding", encoding);
142 
143  string_view tmp = this->self().loadStr();
144  this->self().endTag(tag);
145 
146  if (encoding == "gz-base64") {
147  auto p = Base64::decode(tmp);
148  auto dstLen = uLongf(len); // TODO check for overflow?
149  if ((uncompress(reinterpret_cast<Bytef*>(data), &dstLen,
150  reinterpret_cast<const Bytef*>(p.first.data()), uLong(p.second))
151  != Z_OK) ||
152  (dstLen != len)) {
153  throw MSXException("Error while decompressing blob.");
154  }
155  } else if ((encoding == "hex") || (encoding == "base64")) {
156  bool ok = (encoding == "hex")
157  ? HexDump::decode_inplace(tmp, static_cast<uint8_t*>(data), len)
158  : Base64 ::decode_inplace(tmp, static_cast<uint8_t*>(data), len);
159  if (!ok) {
160  throw XMLException(
161  "Length of decoded blob different from "
162  "expected value (", len, ')');
163  }
164  } else {
165  throw XMLException("Unsupported encoding \"", encoding, "\" for blob");
166  }
167 }
168 
169 template class InputArchiveBase<MemInputArchive>;
170 template class InputArchiveBase<XmlInputArchive>;
171 
173 
174 void MemOutputArchive::save(const std::string& s)
175 {
176  auto size = s.size();
177  byte* buf = buffer.allocate(sizeof(size) + size);
178  memcpy(buf, &size, sizeof(size));
179  memcpy(buf + sizeof(size), s.data(), size);
180 }
181 
183 {
184  return buffer.release(size);
185 }
186 
188 
189 void MemInputArchive::load(std::string& s)
190 {
191  size_t length;
192  load(length);
193  s.resize(length);
194  if (length) {
195  get(&s[0], length);
196  }
197 }
198 
200 {
201  size_t length;
202  load(length);
203  const byte* p = buffer.getCurrentPos();
204  buffer.skip(length);
205  return string_view(reinterpret_cast<const char*>(p), length);
206 }
207 
209 
210 // Too small inputs don't compress very well (often the compressed size is even
211 // bigger than the input). It also takes a relatively long time (because often
212 // compression has a relatively large setup time). I choose this value
213 // semi-arbitrary. I only made it >= 52 so that the (incompressible) RP5C01
214 // registers won't be compressed.
215 static const size_t SMALL_SIZE = 64;
216 void MemOutputArchive::serialize_blob(const char*, const void* data, size_t len,
217  bool diff)
218 {
219  // Delta-compress in-memory blobs, see DeltaBlock.hh for more details.
220  if (len > SMALL_SIZE) {
221  unsigned deltaBlockIdx = unsigned(deltaBlocks.size());
222  save(deltaBlockIdx); // see comment below in MemInputArchive
223  deltaBlocks.push_back(diff
224  ? lastDeltaBlocks.createNew(
225  data, static_cast<const uint8_t*>(data), len)
226  : lastDeltaBlocks.createNullDiff(
227  data, static_cast<const uint8_t*>(data), len));
228  } else {
229  byte* buf = buffer.allocate(len);
230  memcpy(buf, data, len);
231  }
232 
233 }
234 
235 void MemInputArchive::serialize_blob(const char*, void* data, size_t len, bool /*diff*/)
236 {
237  if (len > SMALL_SIZE) {
238  // Usually blobs are saved in the same order as they are loaded
239  // (via the serialize_blob() methods in respectively
240  // MemOutputArchive and MemInputArchive). In that case keeping
241  // track of the deltaBlockIdx in the savestate itself is
242  // redundant (it will simply be an increasing value). However
243  // in rare cases, via the {begin,end,skip)Section() methods, it
244  // is possible that certain blobs are stored in the savestate,
245  // but skipped while loading. That's why we do need the index.
246  unsigned deltaBlockIdx; load(deltaBlockIdx);
247  deltaBlocks[deltaBlockIdx]->apply(static_cast<uint8_t*>(data), len);
248  } else {
249  memcpy(data, buffer.getCurrentPos(), len);
250  buffer.skip(len);
251  }
252 }
253 
255 
256 XmlOutputArchive::XmlOutputArchive(const string& filename)
257  : root("serial")
258 {
259  root.addAttribute("openmsx_version", Version::full());
260  root.addAttribute("date_time", Date::toString(time(nullptr)));
261  root.addAttribute("platform", TARGET_PLATFORM);
262  {
263  auto f = FileOperations::openFile(filename, "wb");
264  if (!f) goto error;
265  int duped_fd = dup(fileno(f.get()));
266  if (duped_fd == -1) goto error;
267  file = gzdopen(duped_fd, "wb9");
268  if (!file) {
269  close(duped_fd);
270  goto error;
271  }
272  current.push_back(&root);
273  return; // success
274  // on scope-exit 'File* f' is closed, and 'gzFile file'
275  // uses the dup()'ed file descriptor.
276  }
277 
278 error:
279  throw XMLException("Could not open compressed file \"", filename, "\"");
280 }
281 
283 {
284  assert(current.back() == &root);
285  const char* header =
286  "<?xml version=\"1.0\" ?>\n"
287  "<!DOCTYPE openmsx-serialize SYSTEM 'openmsx-serialize.dtd'>\n";
288  gzwrite(file, const_cast<char*>(header), unsigned(strlen(header)));
289  string dump = root.dump();
290  gzwrite(file, const_cast<char*>(dump.data()), unsigned(dump.size()));
291  gzclose(file);
292 }
293 
295 {
296  save(string(1, c));
297 }
298 void XmlOutputArchive::save(const string& str)
299 {
300  assert(!current.empty());
301  assert(current.back()->getData().empty());
302  current.back()->setData(str);
303 }
305 {
306  assert(!current.empty());
307  assert(current.back()->getData().empty());
308  current.back()->setData(b ? "true" : "false");
309 }
310 void XmlOutputArchive::save(unsigned char b)
311 {
312  save(unsigned(b));
313 }
314 void XmlOutputArchive::save(signed char c)
315 {
316  save(int(c));
317 }
319 {
320  save(int(c));
321 }
323 {
324  saveImpl(i);
325 }
326 void XmlOutputArchive::save(unsigned u)
327 {
328  saveImpl(u);
329 }
330 void XmlOutputArchive::save(unsigned long long ull)
331 {
332  saveImpl(ull);
333 }
334 
335 void XmlOutputArchive::attribute(const char* name, const string& str)
336 {
337  assert(!current.empty());
338  assert(!current.back()->hasAttribute(name));
339  current.back()->addAttribute(name, str);
340 }
341 void XmlOutputArchive::attribute(const char* name, int i)
342 {
343  attributeImpl(name, i);
344 }
345 void XmlOutputArchive::attribute(const char* name, unsigned u)
346 {
347  attributeImpl(name, u);
348 }
349 
350 void XmlOutputArchive::beginTag(const char* tag)
351 {
352  assert(!current.empty());
353  auto& elem = current.back()->addChild(tag);
354  current.push_back(&elem);
355 }
356 void XmlOutputArchive::endTag(const char* tag)
357 {
358  assert(!current.empty());
359  assert(current.back()->getName() == tag); (void)tag;
360  current.pop_back();
361 }
362 
364 
365 XmlInputArchive::XmlInputArchive(const string& filename)
366  : rootElem(XMLLoader::load(filename, "openmsx-serialize.dtd"))
367 {
368  elems.emplace_back(&rootElem, 0);
369 }
370 
372 {
373  if (!elems.back().first->getChildren().empty()) {
374  throw XMLException("No child tags expected for primitive type");
375  }
376  return elems.back().first->getData();
377 }
378 void XmlInputArchive::load(string& t)
379 {
380  t = loadStr().str();
381 }
383 {
384  std::string str;
385  load(str);
386  std::istringstream is(str);
387  is >> c;
388 }
390 {
391  string_view s = loadStr();
392  if ((s == "true") || (s == "1")) {
393  b = true;
394  } else if ((s == "false") || (s == "0")) {
395  b = false;
396  } else {
397  throw XMLException("Bad value found for boolean: ", s);
398  }
399 }
400 
401 // This function parses a number from a string. It's similar to the generic
402 // templatized XmlInputArchive::load() method, but _much_ faster. It does
403 // have some limitations though:
404 // - it can't handle leading whitespace
405 // - it can't handle extra characters at the end of the string
406 // - it can only handle one base (only decimal, not octal or hexadecimal)
407 // - it doesn't understand a leading '+' sign
408 // - it doesn't detect overflow or underflow (The generic implementation sets
409 // a 'bad' flag on the stream and clips the result to the min/max allowed
410 // value. Though this 'bad' flag was ignored by the openMSX code).
411 // This routine is only used to parse strings we've written ourselves (and the
412 // savestate/replay XML files are not meant to be manually edited). So the
413 // above limitations don't really matter. And we can use the speed gain.
414 template<bool IS_SIGNED> struct ConditionalNegate;
415 template<> struct ConditionalNegate<true> {
416  template<typename T> void operator()(bool negate, T& t) {
417  if (negate) t = -t; // ok to negate a signed type
418  }
419 };
420 template<> struct ConditionalNegate<false> {
421  template<typename T> void operator()(bool negate, T& /*t*/) {
422  assert(!negate); (void)negate; // can't negate unsigned type
423  }
424 };
425 template<typename T> static inline void fastAtoi(string_view str, T& t)
426 {
427  t = 0;
428  bool neg = false;
429  size_t i = 0;
430  size_t l = str.size();
431 
432  static const bool IS_SIGNED = std::numeric_limits<T>::is_signed;
433  if (IS_SIGNED) {
434  if (l == 0) return;
435  if (str[0] == '-') {
436  neg = true;
437  i = 1;
438  }
439  }
440  for (; i < l; ++i) {
441  unsigned d = str[i] - '0';
442  if (unlikely(d > 9)) {
443  throw XMLException("Invalid integer: ", str);
444  }
445  t = 10 * t + d;
446  }
447  // The following stuff does the equivalent of:
448  // if (neg) t = -t;
449  // Though this expression triggers a warning on VC++ when T is an
450  // unsigned type. This complex template stuff avoids the warning.
451  ConditionalNegate<IS_SIGNED> negateFunctor;
452  negateFunctor(neg, t);
453 }
455 {
456  string_view str = loadStr();
457  fastAtoi(str, i);
458 }
459 void XmlInputArchive::load(unsigned& u)
460 {
461  string_view str = loadStr();
462  fastAtoi(str, u);
463 }
464 void XmlInputArchive::load(unsigned long long& ull)
465 {
466  string_view str = loadStr();
467  fastAtoi(str, ull);
468 }
469 void XmlInputArchive::load(unsigned char& b)
470 {
471  unsigned i;
472  load(i);
473  b = i;
474 }
475 void XmlInputArchive::load(signed char& c)
476 {
477  int i;
478  load(i);
479  c = i;
480 }
482 {
483  int i;
484  load(i);
485  c = i;
486 }
487 
488 void XmlInputArchive::beginTag(const char* tag)
489 {
490  auto* child = elems.back().first->findNextChild(
491  tag, elems.back().second);
492  if (!child) {
493  string path;
494  for (auto& e : elems) {
495  strAppend(path, e.first->getName(), '/');
496  }
497  throw XMLException("No child tag \"", tag,
498  "\" found at location \"", path, '\"');
499  }
500  elems.emplace_back(child, 0);
501 }
502 void XmlInputArchive::endTag(const char* tag)
503 {
504  const auto& elem = *elems.back().first;
505  if (elem.getName() != tag) {
506  throw XMLException("End tag \"", elem.getName(),
507  "\" not equal to begin tag \"", tag, "\"");
508  }
509  auto& elem2 = const_cast<XMLElement&>(elem);
510  elem2.clearName(); // mark this elem for later beginTag() calls
511  elems.pop_back();
512 }
513 
514 void XmlInputArchive::attribute(const char* name, string& t)
515 {
516  try {
517  t = elems.back().first->getAttribute(name);
518  } catch (ConfigException& e) {
519  throw XMLException(e.getMessage());
520  }
521 }
522 void XmlInputArchive::attribute(const char* name, int& i)
523 {
524  attributeImpl(name, i);
525 }
526 void XmlInputArchive::attribute(const char* name, unsigned& u)
527 {
528  attributeImpl(name, u);
529 }
530 bool XmlInputArchive::hasAttribute(const char* name)
531 {
532  return elems.back().first->hasAttribute(name);
533 }
534 bool XmlInputArchive::findAttribute(const char* name, unsigned& value)
535 {
536  return elems.back().first->findAttributeInt(name, value);
537 }
539 {
540  return int(elems.back().first->getChildren().size());
541 }
542 
543 } // namespace openmsx
void attribute(const char *name, T &t)
Definition: serialize.hh:823
T length(const vecN< N, T > &x)
Definition: gl_vec.hh:333
bool findAttribute(const char *name, unsigned &value)
Definition: serialize.cc:534
void serialize_blob(const char *, const void *data, size_t len, bool diff=true)
Definition: serialize.cc:216
void operator()(bool negate, T &t)
Definition: serialize.cc:416
#define unlikely(x)
Definition: likely.hh:15
void loadChar(char &c)
Definition: serialize.cc:382
XmlInputArchive(const std::string &filename)
Definition: serialize.cc:365
void * getPointer(unsigned id)
Definition: serialize.cc:115
void serialize_blob(const char *tag, void *data, size_t len, bool diff=true)
Definition: serialize.cc:136
void save(const T &t)
Definition: serialize.hh:616
uint8_t byte
8 bit unsigned integer
Definition: openmsx.hh:26
std::string dump() const
Definition: XMLElement.cc:279
unsigned getId(const void *p) const
Definition: serialize.cc:127
void endTag(const char *tag)
Definition: serialize.cc:502
void addPointer(unsigned id, const void *p)
Definition: serialize.cc:121
MemBuffer< byte > releaseBuffer(size_t &size)
Definition: serialize.cc:182
string_view loadStr()
Definition: serialize.cc:371
bool hasAttribute(const char *name)
Definition: serialize.cc:530
void strAppend(std::string &result, Ts &&...ts)
Definition: strCat.hh:648
void save(SDL_Surface *surface, const std::string &filename)
Definition: PNG.cc:366
const std::string & getMessage() const
Definition: MSXException.hh:23
int countChildren() const
Definition: serialize.cc:538
bool decode_inplace(string_view input, uint8_t *output, size_t outSize)
Definition: Base64.cc:125
void addAttribute(string_view name, string_view value)
Definition: XMLElement.cc:57
void attribute(const char *name, const T &t)
Definition: serialize.hh:760
void endTag(const char *tag)
Definition: serialize.cc:356
void attributeImpl(const char *name, const T &t)
Definition: serialize.hh:756
string_view::const_iterator end(const string_view &x)
Definition: string_view.hh:153
void attribute(const char *name, T &t)
Load/store an attribute from/in the archive.
Definition: serialize.hh:212
void save(const T &t)
Definition: serialize.hh:731
void uncompress(const char *input, size_t inLen, char *output, size_t outLen)
Definition: snappy.cc:166
const T * data() const
Returns pointer to the start of the memory buffer.
Definition: MemBuffer.hh:90
Thanks to enen for testing this on a real cartridge:
Definition: Autofire.cc:5
std::string toString(time_t time)
Definition: Date.cc:152
XMLElement load(string_view filename, string_view systemID)
Definition: XMLLoader.cc:31
bool decode_inplace(string_view input, uint8_t *output, size_t outSize)
Definition: HexDump.cc:72
size_t size() const
void serialize_blob(const char *, void *data, size_t len, bool diff=true)
Definition: serialize.cc:235
string_view loadStr()
Definition: serialize.cc:199
This class implements a (close approximation) of the std::string_view class.
Definition: string_view.hh:15
std::string str() const
Definition: string_view.cc:12
void operator()(bool negate, T &)
Definition: serialize.cc:421
void serialize_blob(const char *tag, const void *data, size_t len, bool diff=true)
Definition: serialize.cc:76
void beginTag(const char *tag)
Definition: serialize.cc:488
uint8_t * data()
static std::string full()
Definition: Version.cc:8
FILE_t openFile(const std::string &filename, const std::string &mode)
Call fopen() in a platform-independent manner.
void saveImpl(const T &t)
Definition: serialize.hh:725
XmlOutputArchive(const std::string &filename)
Definition: serialize.cc:256
size_type size() const
Definition: string_view.hh:52
void beginTag(const char *tag)
Definition: serialize.cc:350
void attributeImpl(const char *name, T &t)
Definition: serialize.hh:816