aboutsummaryrefslogtreecommitdiff
path: root/libbuild2/depdb.cxx
blob: 32e5916e23ed3f7f40d05e330b85540d0debad61 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
// file      : libbuild2/depdb.cxx -*- C++ -*-
// copyright : Copyright (c) 2014-2019 Code Synthesis Ltd
// license   : MIT; see accompanying LICENSE file

#include <libbuild2/depdb.hxx>

#ifdef _WIN32
#  include <libbutl/win32-utility.hxx>
#endif

#include <libbuild2/filesystem.hxx>  // mtime()
#include <libbuild2/diagnostics.hxx>

using namespace std;
using namespace butl;

namespace build2
{
  depdb_base::
  depdb_base (const path& p, timestamp mt)
  {
    fdopen_mode om (fdopen_mode::out | fdopen_mode::binary);
    ifdstream::iostate em (ifdstream::badbit);

    if (mt == timestamp_nonexistent)
    {
      state_ = state::write;
      om |= fdopen_mode::create | fdopen_mode::exclusive;
      em |= ifdstream::failbit;
    }
    else
    {
      state_ = state::read;
      om |= fdopen_mode::in;
    }

    auto_fd fd;
    try
    {
      fd = fdopen (p, om);
    }
    catch (const io_error&)
    {
      bool c (state_ == state::write);

      diag_record dr (fail);
      dr << "unable to " << (c ? "create" : "open") << ' ' << p;

      if (c)
        dr << info << "did you forget to add fsdir{} prerequisite for "
           << "output directory?";

      dr << endf;
    }

    // Open the corresponding stream. Note that if we throw after that, the
    // corresponding member will not be destroyed. This is the reason for the
    // depdb/base split.
    //
    if (state_ == state::read)
    {
      new (&is_) ifdstream (move (fd), em);
      buf_ = static_cast<fdbuf*> (is_.rdbuf ());
    }
    else
    {
      new (&os_) ofdstream (move (fd), em);
      buf_ = static_cast<fdbuf*> (os_.rdbuf ());
    }
  }

  depdb::
  depdb (path_type&& p, timestamp mt)
      : depdb_base (p, mt),
        path (move (p)),
        mtime (mt != timestamp_nonexistent ? mt : timestamp_unknown),
        touch (false)
  {
    // Read/write the database format version.
    //
    if (state_ == state::read)
    {
      string* l (read ());
      if (l == nullptr || *l != "1")
        write ('1');
    }
    else
      write ('1');
  }

  depdb::
  depdb (path_type p)
      : depdb (move (p), build2::mtime (p))
  {
  }

  void depdb::
  change (bool trunc)
  {
    assert (state_ != state::write);

    // Transfer the file descriptor from ifdstream to ofdstream. Note that the
    // steps in this dance must be carefully ordered to make sure we don't
    // call any destructors twice in the face of exceptions.
    //
    auto_fd fd (is_.release ());

    // Consider this scenario: we are overwriting an old line (so it ends with
    // a newline and the "end marker") but the operation failed half way
    // through. Now we have the prefix from the new line, the suffix from the
    // old, and everything looks valid. So what we need is to somehow
    // invalidate the old content so that it can never combine with (partial)
    // new content to form a valid line. One way to do that would be to
    // truncate the file.
    //
    if (trunc)
    try
    {
      fdtruncate (fd.get (), pos_);
    }
    catch (const io_error& e)
    {
      fail << "unable to truncate " << path << ": " << e;
    }

    // Note: the file descriptor position can be beyond the pos_ value due to
    // the ifdstream buffering. That's why we need to seek to switch from
    // reading to writing.
    //
    try
    {
      fdseek (fd.get (), pos_, fdseek_mode::set);
    }
    catch (const io_error& e)
    {
      fail << "unable to rewind " << path << ": " << e;
    }

    // @@ Strictly speaking, ofdstream can throw which will leave us in a
    //    non-destructible state. Unlikely but possible.
    //
    is_.~ifdstream ();
    new (&os_) ofdstream (move (fd),
                          ofdstream::badbit | ofdstream::failbit,
                          pos_);
    buf_ = static_cast<fdbuf*> (os_.rdbuf ());

    state_ = state::write;
    mtime = timestamp_unknown;
  }

  string* depdb::
  read_ ()
  {
    // Save the start position of this line so that we can overwrite it.
    //
    pos_ = buf_->tellg ();

    try
    {
      // Note that we intentionally check for eof after updating the write
      // position.
      //
      if (state_ == state::read_eof)
        return nullptr;

      getline (is_, line_); // Calls line_.erase().

      // The line should always end with a newline. If it doesn't, then this
      // line (and the rest of the database) is assumed corrupted. Also peek
      // at the character after the newline. We should either have the next
      // line or '\0', which is our "end marker", that is, it indicates the
      // database was properly closed.
      //
      ifdstream::int_type c;
      if (is_.fail () || // Nothing got extracted.
          is_.eof ()  || // Eof reached before delimiter.
          (c = is_.peek ()) == ifdstream::traits_type::eof ())
      {
        // Preemptively switch to writing. While we could have delayed this
        // until the user called write(), if the user calls read() again (for
        // whatever misguided reason) we will mess up the overwrite position.
        //
        change ();
        return nullptr;
      }

      // Handle the "end marker". Note that the caller can still switch to the
      // write mode on this line. And, after calling read() again, write to
      // the next line (i.e., start from the "end marker").
      //
      if (c == '\0')
        state_ = state::read_eof;
    }
    catch (const io_error& e)
    {
      fail << "unable to read from " << path << ": " << e;
    }

    return &line_;
  }

  bool depdb::
  skip ()
  {
    if (state_ == state::read_eof)
      return true;

    assert (state_ == state::read);

    // The rest is pretty similar in logic to read_() above.
    //
    pos_ = buf_->tellg ();

    try
    {
      // Keep reading lines checking for the end marker after each newline.
      //
      ifdstream::int_type c;
      do
      {
        if ((c = is_.get ()) == '\n')
        {
          if ((c = is_.get ()) == '\0')
          {
            state_ = state::read_eof;
            return true;
          }
        }
      } while (c != ifdstream::traits_type::eof ());
    }
    catch (const io_error& e)
    {
      fail << "unable to read from " << path << ": " << e;
    }

    // Invalid database so change over to writing.
    //
    change ();
    return false;
  }

  void depdb::
  write (const char* s, size_t n, bool nl)
  {
    // Switch to writing if we are still reading.
    //
    if (state_ != state::write)
      change ();

    try
    {
      os_.write (s, static_cast<streamsize> (n));

      if (nl)
        os_.put ('\n');
    }
    catch (const io_error& e)
    {
      fail << "unable to write to " << path << ": " << e;
    }
  }

  void depdb::
  write (char c, bool nl)
  {
    // Switch to writing if we are still reading.
    //
    if (state_ != state::write)
      change ();

    try
    {
      os_.put (c);

      if (nl)
        os_.put ('\n');
    }
    catch (const io_error& e)
    {
      fail << "unable to write to " << path << ": " << e;
    }
  }

  void depdb::
  close ()
  {
    // If we are at eof, then it means all lines are good, there is the "end
    // marker" at the end, and we don't need to do anything, except, maybe
    // touch the file. Otherwise, if we are still in the read mode, truncate
    // the rest, and then add the "end marker" (we cannot have anything in the
    // write mode since we truncate in change()).
    //
    if (state_ == state::read_eof)
    {
      if (!touch)
      try
      {
        is_.close ();
        return;
      }
      catch (const io_error& e)
      {
        fail << "unable to close " << path << ": " << e;
      }

      // While there are utime(2)/utimensat(2) (and probably something similar
      // for Windows), for now we just overwrite the "end marker". Hopefully
      // no implementation will be smart enough to recognize this is a no-op
      // and skip updating mtime (which would probably be incorrect, spec-
      // wise). And this could even be faster since we already have the file
      // descriptor. Or it might be slower since so far we've only been
      // reading.
      //
      pos_ = buf_->tellg ();         // The last line is accepted.
      change (false /* truncate */); // Write end marker below.
    }
    else if (state_ != state::write)
    {
      pos_ = buf_->tellg (); // The last line is accepted.
      change (true /* truncate */);
    }

    if (mtime_check ())
      start_ = system_clock::now ();

    try
    {
      os_.put ('\0'); // The "end marker".
      os_.close ();
    }
    catch (const io_error& e)
    {
      fail << "unable to flush " << path << ": " << e;
    }

    // On some platforms (currently confirmed on FreeBSD running as VMs) one
    // can sometimes end up with a modification time that is a bit after the
    // call to close(). And in some tight cases this can mess with our
    // "protocol" that a valid depdb should be no older than the target it is
    // for.
    //
    // Note that this does not seem to be related to clock adjustments but
    // rather feels like the modification time is set when the changes
    // actually hit some lower-level layer (e.g., OS or filesystem
    // driver). One workaround that appears to work is to query the
    // mtime. This seems to force that layer to commit to a timestamp.
    //
#if defined(__FreeBSD__)
    mtime = build2::mtime (path); // Save for debugging/check below.
#endif
  }

  void depdb::
  check_mtime_ (const path_type& t, timestamp e)
  {
    // We could call the static version but then we would have lost additional
    // information for some platforms.
    //
    timestamp t_mt (build2::mtime (t));
    timestamp d_mt (build2::mtime (path));

    if (d_mt > t_mt)
    {
      if (e == timestamp_unknown)
        e = system_clock::now ();

      fail << "backwards modification times detected:\n"
           << "    " << start_ << " sequence start\n"
#if defined(__FreeBSD__)
           << "    " << mtime  << " close mtime\n"
#endif
           << "    " << d_mt   << " " << path.string () << '\n'
           << "    " << t_mt   << " " << t.string () << '\n'
           << "    " << e      << " sequence end";
    }
  }

  void depdb::
  check_mtime_ (timestamp s,
                const path_type& d,
                const path_type& t,
                timestamp e)
  {
    using build2::mtime;

    timestamp t_mt (mtime (t));
    timestamp d_mt (mtime (d));

    if (d_mt > t_mt)
    {
      fail << "backwards modification times detected:\n"
           << "    " << s    << " sequence start\n"
           << "    " << d_mt << " " << d.string () << '\n'
           << "    " << t_mt << " " << t.string () << '\n'
           << "    " << e    << " sequence end";
    }
  }
}