openMSX
serialize.cc
Go to the documentation of this file.
1 #include "serialize.hh"
2 #include "Base64.hh"
3 #include "HexDump.hh"
4 #include "XMLLoader.hh"
5 #include "XMLElement.hh"
6 #include "ConfigException.hh"
7 #include "XMLException.hh"
8 #include "DeltaBlock.hh"
9 #include "MemBuffer.hh"
10 #include "FileOperations.hh"
11 #include "Version.hh"
12 #include "Date.hh"
13 #include "cstdiop.hh" // for dup()
14 #include <cstring>
15 #include <limits>
16 
17 using std::string;
18 
19 namespace openmsx {
20 
21 template<typename Derived>
22 void ArchiveBase<Derived>::attribute(const char* name, const char* value)
23 {
24  string valueStr(value);
25  self().attribute(name, valueStr);
26 }
27 template class ArchiveBase<MemOutputArchive>;
28 template class ArchiveBase<XmlOutputArchive>;
29 
31 
32 unsigned OutputArchiveBase2::generateID1(const void* p)
33 {
34  #ifdef linux
35  assert("Can't serialize ID of object located on the stack" &&
36  !addressOnStack(p));
37  #endif
38  ++lastId;
39  assert(polyIdMap.find(p) == end(polyIdMap));
40  polyIdMap[p] = lastId;
41  return lastId;
42 }
43 unsigned OutputArchiveBase2::generateID2(
44  const void* p, const std::type_info& typeInfo)
45 {
46  #ifdef linux
47  assert("Can't serialize ID of object located on the stack" &&
48  !addressOnStack(p));
49  #endif
50  ++lastId;
51  auto key = std::make_pair(p, std::type_index(typeInfo));
52  assert(idMap.find(key) == end(idMap));
53  idMap[key] = lastId;
54  return lastId;
55 }
56 
57 unsigned OutputArchiveBase2::getID1(const void* p)
58 {
59  auto it = polyIdMap.find(p);
60  return it != end(polyIdMap) ? it->second : 0;
61 }
62 unsigned OutputArchiveBase2::getID2(
63  const void* p, const std::type_info& typeInfo)
64 {
65  auto it = idMap.find({p, std::type_index(typeInfo)});
66  return it != end(idMap) ? it->second : 0;
67 }
68 
69 
70 template<typename Derived>
72  const char* tag, const void* data_, size_t len, bool /*diff*/)
73 {
74  auto* data = static_cast<const uint8_t*>(data_);
75 
76  string encoding;
77  string tmp;
78  if (false) {
79  // useful for debugging
80  encoding = "hex";
81  tmp = HexDump::encode(data, len);
82  } else if (false) {
83  encoding = "base64";
84  tmp = Base64::encode(data, len);
85  } else {
86  encoding = "gz-base64";
87  // TODO check for overflow?
88  auto dstLen = uLongf(len + len / 1000 + 12 + 1); // worst-case
89  MemBuffer<byte> buf(dstLen);
90  if (compress2(buf.data(), &dstLen,
91  reinterpret_cast<const Bytef*>(data),
92  uLong(len), 9)
93  != Z_OK) {
94  throw MSXException("Error while compressing blob.");
95  }
96  tmp = Base64::encode(buf.data(), dstLen);
97  }
98  this->self().beginTag(tag);
99  this->self().attribute("encoding", encoding);
100  Saver<string> saver;
101  saver(this->self(), tmp, false);
102  this->self().endTag(tag);
103 }
104 
107 
109 
111 {
112  auto it = idMap.find(id);
113  return it != end(idMap) ? it->second : nullptr;
114 }
115 
116 void InputArchiveBase2::addPointer(unsigned id, const void* p)
117 {
118  assert(idMap.find(id) == end(idMap));
119  idMap[id] = const_cast<void*>(p);
120 }
121 
122 unsigned InputArchiveBase2::getId(const void* ptr) const
123 {
124  for (const auto& p : idMap) {
125  if (p.second == ptr) return p.first;
126  }
127  return 0;
128 }
129 
130 template<typename Derived>
132  const char* tag, void* data, size_t len, bool /*diff*/)
133 {
134  this->self().beginTag(tag);
135  string encoding;
136  this->self().attribute("encoding", encoding);
137 
138  string_view tmp = this->self().loadStr();
139  this->self().endTag(tag);
140 
141  if (encoding == "gz-base64") {
142  auto p = Base64::decode(tmp);
143  auto dstLen = uLongf(len); // TODO check for overflow?
144  if ((uncompress(reinterpret_cast<Bytef*>(data), &dstLen,
145  reinterpret_cast<const Bytef*>(p.first.data()), uLong(p.second))
146  != Z_OK) ||
147  (dstLen != len)) {
148  throw MSXException("Error while decompressing blob.");
149  }
150  } else if ((encoding == "hex") || (encoding == "base64")) {
151  bool ok = (encoding == "hex")
152  ? HexDump::decode_inplace(tmp, static_cast<uint8_t*>(data), len)
153  : Base64 ::decode_inplace(tmp, static_cast<uint8_t*>(data), len);
154  if (!ok) {
155  throw XMLException(
156  "Length of decoded blob different from "
157  "expected value (", len, ')');
158  }
159  } else {
160  throw XMLException("Unsupported encoding \"", encoding, "\" for blob");
161  }
162 }
163 
164 template class InputArchiveBase<MemInputArchive>;
165 template class InputArchiveBase<XmlInputArchive>;
166 
168 
169 void MemOutputArchive::save(const std::string& s)
170 {
171  auto size = s.size();
172  byte* buf = buffer.allocate(sizeof(size) + size);
173  memcpy(buf, &size, sizeof(size));
174  memcpy(buf + sizeof(size), s.data(), size);
175 }
176 
178 {
179  return buffer.release(size);
180 }
181 
183 
184 void MemInputArchive::load(std::string& s)
185 {
186  size_t length;
187  load(length);
188  s.resize(length);
189  if (length) {
190  get(&s[0], length);
191  }
192 }
193 
195 {
196  size_t length;
197  load(length);
198  const byte* p = buffer.getCurrentPos();
199  buffer.skip(length);
200  return string_view(reinterpret_cast<const char*>(p), length);
201 }
202 
204 
205 // Too small inputs don't compress very well (often the compressed size is even
206 // bigger than the input). It also takes a relatively long time (because often
207 // compression has a relatively large setup time). I choose this value
208 // semi-arbitrary. I only made it >= 52 so that the (incompressible) RP5C01
209 // registers won't be compressed.
210 static const size_t SMALL_SIZE = 64;
211 void MemOutputArchive::serialize_blob(const char* /*tag*/, const void* data,
212  size_t len, bool diff)
213 {
214  // Delta-compress in-memory blobs, see DeltaBlock.hh for more details.
215  if (len > SMALL_SIZE) {
216  auto deltaBlockIdx = unsigned(deltaBlocks.size());
217  save(deltaBlockIdx); // see comment below in MemInputArchive
218  deltaBlocks.push_back(diff
219  ? lastDeltaBlocks.createNew(
220  data, static_cast<const uint8_t*>(data), len)
221  : lastDeltaBlocks.createNullDiff(
222  data, static_cast<const uint8_t*>(data), len));
223  } else {
224  byte* buf = buffer.allocate(len);
225  memcpy(buf, data, len);
226  }
227 
228 }
229 
230 void MemInputArchive::serialize_blob(const char* /*tag*/, void* data,
231  size_t len, bool /*diff*/)
232 {
233  if (len > SMALL_SIZE) {
234  // Usually blobs are saved in the same order as they are loaded
235  // (via the serialize_blob() methods in respectively
236  // MemOutputArchive and MemInputArchive). In that case keeping
237  // track of the deltaBlockIdx in the savestate itself is
238  // redundant (it will simply be an increasing value). However
239  // in rare cases, via the {begin,end,skip)Section() methods, it
240  // is possible that certain blobs are stored in the savestate,
241  // but skipped while loading. That's why we do need the index.
242  unsigned deltaBlockIdx; load(deltaBlockIdx);
243  deltaBlocks[deltaBlockIdx]->apply(static_cast<uint8_t*>(data), len);
244  } else {
245  memcpy(data, buffer.getCurrentPos(), len);
246  buffer.skip(len);
247  }
248 }
249 
251 
252 XmlOutputArchive::XmlOutputArchive(const string& filename)
253  : root("serial")
254 {
255  root.addAttribute("openmsx_version", Version::full());
256  root.addAttribute("date_time", Date::toString(time(nullptr)));
257  root.addAttribute("platform", TARGET_PLATFORM);
258  {
259  auto f = FileOperations::openFile(filename, "wb");
260  if (!f) goto error;
261  int duped_fd = dup(fileno(f.get()));
262  if (duped_fd == -1) goto error;
263  file = gzdopen(duped_fd, "wb9");
264  if (!file) {
265  close(duped_fd);
266  goto error;
267  }
268  current.push_back(&root);
269  return; // success
270  // on scope-exit 'File* f' is closed, and 'gzFile file'
271  // uses the dup()'ed file descriptor.
272  }
273 
274 error:
275  throw XMLException("Could not open compressed file \"", filename, "\"");
276 }
277 
279 {
280  assert(current.back() == &root);
281  const char* header =
282  "<?xml version=\"1.0\" ?>\n"
283  "<!DOCTYPE openmsx-serialize SYSTEM 'openmsx-serialize.dtd'>\n";
284  gzwrite(file, const_cast<char*>(header), unsigned(strlen(header)));
285  string dump = root.dump();
286  gzwrite(file, const_cast<char*>(dump.data()), unsigned(dump.size()));
287  gzclose(file);
288 }
289 
291 {
292  save(string(1, c));
293 }
294 void XmlOutputArchive::save(const string& str)
295 {
296  assert(!current.empty());
297  assert(current.back()->getData().empty());
298  current.back()->setData(str);
299 }
301 {
302  assert(!current.empty());
303  assert(current.back()->getData().empty());
304  current.back()->setData(b ? "true" : "false");
305 }
306 void XmlOutputArchive::save(unsigned char b)
307 {
308  save(unsigned(b));
309 }
310 void XmlOutputArchive::save(signed char c)
311 {
312  save(int(c));
313 }
315 {
316  save(int(c));
317 }
319 {
320  saveImpl(i);
321 }
322 void XmlOutputArchive::save(unsigned u)
323 {
324  saveImpl(u);
325 }
326 void XmlOutputArchive::save(unsigned long long ull)
327 {
328  saveImpl(ull);
329 }
330 
331 void XmlOutputArchive::attribute(const char* name, const string& str)
332 {
333  assert(!current.empty());
334  assert(!current.back()->hasAttribute(name));
335  current.back()->addAttribute(name, str);
336 }
337 void XmlOutputArchive::attribute(const char* name, int i)
338 {
339  attributeImpl(name, i);
340 }
341 void XmlOutputArchive::attribute(const char* name, unsigned u)
342 {
343  attributeImpl(name, u);
344 }
345 
346 void XmlOutputArchive::beginTag(const char* tag)
347 {
348  assert(!current.empty());
349  auto& elem = current.back()->addChild(tag);
350  current.push_back(&elem);
351 }
352 void XmlOutputArchive::endTag(const char* tag)
353 {
354  assert(!current.empty());
355  assert(current.back()->getName() == tag); (void)tag;
356  current.pop_back();
357 }
358 
360 
361 XmlInputArchive::XmlInputArchive(const string& filename)
362  : rootElem(XMLLoader::load(filename, "openmsx-serialize.dtd"))
363 {
364  elems.emplace_back(&rootElem, 0);
365 }
366 
368 {
369  if (!elems.back().first->getChildren().empty()) {
370  throw XMLException("No child tags expected for primitive type");
371  }
372  return elems.back().first->getData();
373 }
374 void XmlInputArchive::load(string& t)
375 {
376  t = loadStr().str();
377 }
379 {
380  std::string str;
381  load(str);
382  std::istringstream is(str);
383  is >> c;
384 }
386 {
387  string_view s = loadStr();
388  if ((s == "true") || (s == "1")) {
389  b = true;
390  } else if ((s == "false") || (s == "0")) {
391  b = false;
392  } else {
393  throw XMLException("Bad value found for boolean: ", s);
394  }
395 }
396 
397 // This function parses a number from a string. It's similar to the generic
398 // templatized XmlInputArchive::load() method, but _much_ faster. It does
399 // have some limitations though:
400 // - it can't handle leading whitespace
401 // - it can't handle extra characters at the end of the string
402 // - it can only handle one base (only decimal, not octal or hexadecimal)
403 // - it doesn't understand a leading '+' sign
404 // - it doesn't detect overflow or underflow (The generic implementation sets
405 // a 'bad' flag on the stream and clips the result to the min/max allowed
406 // value. Though this 'bad' flag was ignored by the openMSX code).
407 // This routine is only used to parse strings we've written ourselves (and the
408 // savestate/replay XML files are not meant to be manually edited). So the
409 // above limitations don't really matter. And we can use the speed gain.
410 template<bool IS_SIGNED> struct ConditionalNegate;
411 template<> struct ConditionalNegate<true> {
412  template<typename T> void operator()(bool negate, T& t) {
413  if (negate) t = -t; // ok to negate a signed type
414  }
415 };
416 template<> struct ConditionalNegate<false> {
417  template<typename T> void operator()(bool negate, T& /*t*/) {
418  assert(!negate); (void)negate; // can't negate unsigned type
419  }
420 };
421 template<typename T> static inline void fastAtoi(string_view str, T& t)
422 {
423  t = 0;
424  bool neg = false;
425  size_t i = 0;
426  size_t l = str.size();
427 
428  static const bool IS_SIGNED = std::numeric_limits<T>::is_signed;
429  if (IS_SIGNED) {
430  if (l == 0) return;
431  if (str[0] == '-') {
432  neg = true;
433  i = 1;
434  }
435  }
436  for (; i < l; ++i) {
437  unsigned d = str[i] - '0';
438  if (unlikely(d > 9)) {
439  throw XMLException("Invalid integer: ", str);
440  }
441  t = 10 * t + d;
442  }
443  // The following stuff does the equivalent of:
444  // if (neg) t = -t;
445  // Though this expression triggers a warning on VC++ when T is an
446  // unsigned type. This complex template stuff avoids the warning.
447  ConditionalNegate<IS_SIGNED> negateFunctor;
448  negateFunctor(neg, t);
449 }
451 {
452  string_view str = loadStr();
453  fastAtoi(str, i);
454 }
455 void XmlInputArchive::load(unsigned& u)
456 {
457  string_view str = loadStr();
458  fastAtoi(str, u);
459 }
460 void XmlInputArchive::load(unsigned long long& ull)
461 {
462  string_view str = loadStr();
463  fastAtoi(str, ull);
464 }
465 void XmlInputArchive::load(unsigned char& b)
466 {
467  unsigned i;
468  load(i);
469  b = i;
470 }
471 void XmlInputArchive::load(signed char& c)
472 {
473  int i;
474  load(i);
475  c = i;
476 }
478 {
479  int i;
480  load(i);
481  c = i;
482 }
483 
484 void XmlInputArchive::beginTag(const char* tag)
485 {
486  auto* child = elems.back().first->findNextChild(
487  tag, elems.back().second);
488  if (!child) {
489  string path;
490  for (auto& e : elems) {
491  strAppend(path, e.first->getName(), '/');
492  }
493  throw XMLException("No child tag \"", tag,
494  "\" found at location \"", path, '\"');
495  }
496  elems.emplace_back(child, 0);
497 }
498 void XmlInputArchive::endTag(const char* tag)
499 {
500  const auto& elem = *elems.back().first;
501  if (elem.getName() != tag) {
502  throw XMLException("End tag \"", elem.getName(),
503  "\" not equal to begin tag \"", tag, "\"");
504  }
505  auto& elem2 = const_cast<XMLElement&>(elem);
506  elem2.clearName(); // mark this elem for later beginTag() calls
507  elems.pop_back();
508 }
509 
510 void XmlInputArchive::attribute(const char* name, string& t)
511 {
512  try {
513  t = elems.back().first->getAttribute(name);
514  } catch (ConfigException& e) {
515  throw XMLException(std::move(e).getMessage());
516  }
517 }
518 void XmlInputArchive::attribute(const char* name, int& i)
519 {
520  attributeImpl(name, i);
521 }
522 void XmlInputArchive::attribute(const char* name, unsigned& u)
523 {
524  attributeImpl(name, u);
525 }
526 bool XmlInputArchive::hasAttribute(const char* name)
527 {
528  return elems.back().first->hasAttribute(name);
529 }
530 bool XmlInputArchive::findAttribute(const char* name, unsigned& value)
531 {
532  return elems.back().first->findAttributeInt(name, value);
533 }
535 {
536  return int(elems.back().first->getChildren().size());
537 }
538 
539 } // namespace openmsx
void attribute(const char *name, T &t)
Definition: serialize.hh:823
T length(const vecN< N, T > &x)
Definition: gl_vec.hh:334
bool findAttribute(const char *name, unsigned &value)
Definition: serialize.cc:530
void operator()(bool negate, T &t)
Definition: serialize.cc:412
#define unlikely(x)
Definition: likely.hh:15
void loadChar(char &c)
Definition: serialize.cc:378
XmlInputArchive(const std::string &filename)
Definition: serialize.cc:361
void * getPointer(unsigned id)
Definition: serialize.cc:110
void serialize_blob(const char *tag, void *data, size_t len, bool diff=true)
Definition: serialize.cc:131
void save(const T &t)
Definition: serialize.hh:616
uint8_t byte
8 bit unsigned integer
Definition: openmsx.hh:26
std::string dump() const
Definition: XMLElement.cc:279
unsigned getId(const void *p) const
Definition: serialize.cc:122
void endTag(const char *tag)
Definition: serialize.cc:498
void addPointer(unsigned id, const void *p)
Definition: serialize.cc:116
MemBuffer< byte > releaseBuffer(size_t &size)
Definition: serialize.cc:177
string_view loadStr()
Definition: serialize.cc:367
bool hasAttribute(const char *name)
Definition: serialize.cc:526
void strAppend(std::string &result, Ts &&...ts)
Definition: strCat.hh:648
size_t size(string_view utf8)
int countChildren() const
Definition: serialize.cc:534
bool decode_inplace(string_view input, uint8_t *output, size_t outSize)
Definition: Base64.cc:125
void addAttribute(string_view name, string_view value)
Definition: XMLElement.cc:57
void attribute(const char *name, const T &t)
Definition: serialize.hh:760
void endTag(const char *tag)
Definition: serialize.cc:352
void attributeImpl(const char *name, const T &t)
Definition: serialize.hh:756
string_view::const_iterator end(const string_view &x)
Definition: string_view.hh:153
void attribute(const char *name, T &t)
Load/store an attribute from/in the archive.
Definition: serialize.hh:212
void save(const T &t)
Definition: serialize.hh:731
void uncompress(const char *input, size_t inLen, char *output, size_t outLen)
Definition: snappy.cc:166
const T * data() const
Returns pointer to the start of the memory buffer.
Definition: MemBuffer.hh:90
Thanks to enen for testing this on a real cartridge:
Definition: Autofire.cc:5
std::string toString(time_t time)
Definition: Date.cc:152
XMLElement load(string_view filename, string_view systemID)
Definition: XMLLoader.cc:31
bool decode_inplace(string_view input, uint8_t *output, size_t outSize)
Definition: HexDump.cc:72
string_view loadStr()
Definition: serialize.cc:194
void serialize_blob(const char *tag, void *data, size_t len, bool diff=true)
Definition: serialize.cc:230
This class implements a (close approximation) of the std::string_view class.
Definition: string_view.hh:15
void serialize_blob(const char *tag, const void *data, size_t len, bool diff=true)
Definition: serialize.cc:211
std::string str() const
Definition: string_view.cc:12
void operator()(bool negate, T &)
Definition: serialize.cc:417
void serialize_blob(const char *tag, const void *data, size_t len, bool diff=true)
Definition: serialize.cc:71
void beginTag(const char *tag)
Definition: serialize.cc:484
static std::string full()
Definition: Version.cc:8
FILE_t openFile(const std::string &filename, const std::string &mode)
Call fopen() in a platform-independent manner.
void saveImpl(const T &t)
Definition: serialize.hh:725
XmlOutputArchive(const std::string &filename)
Definition: serialize.cc:252
size_type size() const
Definition: string_view.hh:52
void beginTag(const char *tag)
Definition: serialize.cc:346
void attributeImpl(const char *name, T &t)
Definition: serialize.hh:816