GCC Code Coverage Report
Directory: . Exec Total Coverage
File: src/parser/bounded_token_buffer.cpp Lines: 88 167 52.7 %
Date: 2021-03-22 Branches: 25 252 9.9 %

Line Exec Source
1
/*********************                                                        */
2
/*! \file bounded_token_buffer.cpp
3
 ** \verbatim
4
 ** Top contributors (to current version):
5
 **   Christopher L. Conway, Morgan Deters, Tim King
6
 ** This file is part of the CVC4 project.
7
 ** Copyright (c) 2009-2021 by the authors listed in the file AUTHORS
8
 ** in the top-level source directory and their institutional affiliations.
9
 ** All rights reserved.  See the file COPYING in the top-level source
10
 ** directory for licensing information.\endverbatim
11
 **
12
 ** \brief An ANTLR3 bounded token stream implementation.
13
 **
14
 ** An ANTLR3 bounded token stream implementation.
15
 ** This code is largely based on the original token buffer implementation
16
 ** in libantlr3c, by Jim Idle.
17
 **/
18
19
/// \file
20
/// Default implementation of CommonTokenStream
21
///
22
23
// [The "BSD licence"]
24
// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC
25
// http://www.temporal-wave.com
26
// http://www.linkedin.com/in/jimidle
27
//
28
// All rights reserved.
29
//
30
// Redistribution and use in source and binary forms, with or without
31
// modification, are permitted provided that the following conditions
32
// are met:
33
// 1. Redistributions of source code must retain the above copyright
34
//    notice, this list of conditions and the following disclaimer.
35
// 2. Redistributions in binary form must reproduce the above copyright
36
//    notice, this list of conditions and the following disclaimer in the
37
//    documentation and/or other materials provided with the distribution.
38
// 3. The name of the author may not be used to endorse or promote products
39
//    derived from this software without specific prior written permission.
40
//
41
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
42
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
43
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
44
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
45
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
46
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
47
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
48
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
49
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
50
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
51
52
#include "parser/bounded_token_buffer.h"
53
54
#include <antlr3commontoken.h>
55
#include <antlr3lexer.h>
56
#include <antlr3tokenstream.h>
57
58
#include "base/check.h"
59
60
namespace CVC4 {
61
namespace parser {
62
63
#ifdef	ANTLR3_WINDOWS
64
#pragma warning( disable : 4100 )
65
#endif
66
67
// TOKEN_STREAM API
68
//
69
static pANTLR3_COMMON_TOKEN tokLT				(pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k);
70
static pANTLR3_COMMON_TOKEN dbgTokLT			(pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k);
71
static pANTLR3_COMMON_TOKEN get					(pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i);
72
static pANTLR3_TOKEN_SOURCE getTokenSource		(pANTLR3_TOKEN_STREAM ts);
73
static void					setTokenSource		(pANTLR3_TOKEN_STREAM ts, pANTLR3_TOKEN_SOURCE tokenSource);
74
static pANTLR3_STRING	    toString			(pANTLR3_TOKEN_STREAM ts);
75
static pANTLR3_STRING	    toStringSS			(pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);
76
static pANTLR3_STRING	    toStringTT			(pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop);
77
static void					setDebugListener	(pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger);
78
79
// INT STREAM API
80
//
81
static void					consume						(pANTLR3_INT_STREAM is);
82
static void					dbgConsume					(pANTLR3_INT_STREAM is);
83
static ANTLR3_UINT32	    _LA							(pANTLR3_INT_STREAM is, ANTLR3_INT32 i);
84
static ANTLR3_UINT32	    dbgLA						(pANTLR3_INT_STREAM is, ANTLR3_INT32 i);
85
static ANTLR3_MARKER	    mark						(pANTLR3_INT_STREAM is);
86
static ANTLR3_MARKER	    dbgMark						(pANTLR3_INT_STREAM is);
87
static void					release						(pANTLR3_INT_STREAM is, ANTLR3_MARKER mark);
88
static ANTLR3_UINT32	    size						(pANTLR3_INT_STREAM is);
89
static ANTLR3_MARKER		tindex						(pANTLR3_INT_STREAM is);
90
static void					rewindStream				(pANTLR3_INT_STREAM is, ANTLR3_MARKER marker);
91
static void					dbgRewindStream				(pANTLR3_INT_STREAM is, ANTLR3_MARKER marker);
92
static void					rewindLast					(pANTLR3_INT_STREAM is);
93
static void					dbgRewindLast				(pANTLR3_INT_STREAM is);
94
static void					seek						(pANTLR3_INT_STREAM is, ANTLR3_MARKER index);
95
static void					dbgSeek						(pANTLR3_INT_STREAM is, ANTLR3_MARKER index);
96
static pANTLR3_STRING		getSourceName				(pANTLR3_INT_STREAM is);
97
98
static pANTLR3_COMMON_TOKEN nextToken(pBOUNDED_TOKEN_BUFFER buffer);
99
100
void
101
5764
BoundedTokenBufferFree(pBOUNDED_TOKEN_BUFFER buffer) {
102
5764
  buffer->commonTstream->free(buffer->commonTstream);
103
5764
  ANTLR3_FREE(buffer->tokenBuffer);
104
5764
  ANTLR3_FREE(buffer);
105
5764
}
106
107
108
/*ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
109
antlr3CommonTokenDebugStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source, pANTLR3_DEBUG_EVENT_LISTENER debugger)
110
{
111
    pANTLR3_COMMON_TOKEN_STREAM	stream;
112
113
	// Create a standard token stream
114
	//
115
	stream = antlr3CommonTokenStreamSourceNew(hint, source);
116
117
	// Install the debugger object
118
	//
119
	stream->tstream->debugger = debugger;
120
121
	// Override standard token stream methods with debugging versions
122
	//
123
	stream->tstream->initialStreamState	= ANTLR3_FALSE;
124
125
	stream->tstream->_LT				= dbgTokLT;
126
127
	stream->tstream->istream->consume		= dbgConsume;
128
	stream->tstream->istream->_LA			= dbgLA;
129
	stream->tstream->istream->mark			= dbgMark;
130
	stream->tstream->istream->rewind		= dbgRewindStream;
131
	stream->tstream->istream->rewindLast	= dbgRewindLast;
132
	stream->tstream->istream->seek			= dbgSeek;
133
134
	return stream;
135
}*/
136
137
5764
pBOUNDED_TOKEN_BUFFER BoundedTokenBufferSourceNew(ANTLR3_UINT32 k,
138
                                                  pANTLR3_TOKEN_SOURCE source) {
139
  pBOUNDED_TOKEN_BUFFER buffer;
140
  pANTLR3_COMMON_TOKEN_STREAM stream;
141
142
5764
  Assert(k > 0);
143
144
  /* Memory for the interface structure */
145
5764
  buffer =
146
5764
      (pBOUNDED_TOKEN_BUFFER)ANTLR3_MALLOC(sizeof(BOUNDED_TOKEN_BUFFER_struct));
147
148
5764
  if (buffer == NULL) {
149
    return NULL;
150
  }
151
152
5764
  buffer->tokenBuffer = (pANTLR3_COMMON_TOKEN*)ANTLR3_MALLOC(
153
      2 * k * sizeof(pANTLR3_COMMON_TOKEN));
154
5764
  if (buffer->tokenBuffer == NULL) {
155
    ANTLR3_FREE(buffer);
156
    return NULL;
157
  }
158
159
5764
  buffer->currentIndex = 0;
160
5764
  buffer->maxIndex = 0;
161
5764
  buffer->k = k;
162
5764
  buffer->bufferSize = 2 * k;
163
5764
  buffer->empty = ANTLR3_TRUE;
164
5764
  buffer->done = ANTLR3_FALSE;
165
166
5764
  stream = antlr3CommonTokenStreamSourceNew(k, source);
167
5764
  if (stream == NULL) {
168
    ANTLR3_FREE(buffer->tokenBuffer);
169
    ANTLR3_FREE(buffer);
170
    return NULL;
171
  }
172
173
5764
  stream->super = buffer;
174
5764
  buffer->commonTstream = stream;
175
176
  /* Defaults */
177
5764
  stream->p = -1;
178
179
  /* Install the token stream API */
180
5764
  stream->tstream->_LT = tokLT;
181
5764
  stream->tstream->get = get;
182
5764
  stream->tstream->getTokenSource = getTokenSource;
183
5764
  stream->tstream->setTokenSource = setTokenSource;
184
5764
  stream->tstream->toString = toString;
185
5764
  stream->tstream->toStringSS = toStringSS;
186
5764
  stream->tstream->toStringTT = toStringTT;
187
5764
  stream->tstream->setDebugListener = setDebugListener;
188
189
  /* Install INT_STREAM interface */
190
5764
  stream->tstream->istream->_LA = _LA;
191
5764
  stream->tstream->istream->mark = mark;
192
5764
  stream->tstream->istream->release = release;
193
5764
  stream->tstream->istream->size = size;
194
5764
  stream->tstream->istream->index = tindex;
195
5764
  stream->tstream->istream->rewind = rewindStream;
196
5764
  stream->tstream->istream->rewindLast = rewindLast;
197
5764
  stream->tstream->istream->seek = seek;
198
5764
  stream->tstream->istream->consume = consume;
199
5764
  stream->tstream->istream->getSourceName = getSourceName;
200
201
5764
  return buffer;
202
}
203
204
// Install a debug listener adn switch to debug mode methods
205
//
206
static void
207
setDebugListener	(pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger)
208
{
209
		// Install the debugger object
210
	//
211
	ts->debugger = debugger;
212
213
	// Override standard token stream methods with debugging versions
214
	//
215
	ts->initialStreamState	= ANTLR3_FALSE;
216
217
	ts->_LT				= dbgTokLT;
218
219
	ts->istream->consume		= dbgConsume;
220
	ts->istream->_LA			= dbgLA;
221
	ts->istream->mark			= dbgMark;
222
	ts->istream->rewind			= dbgRewindStream;
223
	ts->istream->rewindLast		= dbgRewindLast;
224
	ts->istream->seek			= dbgSeek;
225
}
226
227
/** Get the ith token from the current position 1..n where k=1 is the
228
*  first symbol of lookahead.
229
*/
230
91554307
static pANTLR3_COMMON_TOKEN tokLT(pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k) {
231
  pANTLR3_COMMON_TOKEN_STREAM cts;
232
  pBOUNDED_TOKEN_BUFFER buffer;
233
234
91554307
  cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
235
91554307
  buffer = (pBOUNDED_TOKEN_BUFFER) cts->super;
236
237
  /* k must be in the range [-buffer->k..buffer->k] */
238
91554307
  Assert(k <= (ANTLR3_INT32)buffer->k && -k <= (ANTLR3_INT32)buffer->k);
239
240
91554307
  if(k == 0) {
241
    return NULL;
242
  }
243
244
  /* Initialize the buffer on our first call. */
245
91554307
  if( __builtin_expect( (buffer->empty == ANTLR3_TRUE), false ) ) {
246
5752
    Assert(buffer->tokenBuffer != NULL);
247
5752
    buffer->tokenBuffer[ 0 ] = nextToken(buffer);
248
5742
    buffer->maxIndex = 0;
249
5742
    buffer->currentIndex = 0;
250
5742
    buffer->empty = ANTLR3_FALSE;
251
  }
252
253
  ANTLR3_UINT32 kIndex;
254
91554297
  if( k > 0 ) {
255
    /* look-ahead token k is at offset k-1 */
256
91554297
    kIndex = buffer->currentIndex + k - 1;
257
  } else {
258
    /* Can't look behind more tokens than we've consumed. */
259
    Assert(-k <= (ANTLR3_INT32)buffer->currentIndex);
260
    /* look-behind token k is at offset -k */
261
    kIndex = buffer->currentIndex + k;
262
  }
263
264
126017675
  while( kIndex > buffer->maxIndex ) {
265
17231691
    buffer->maxIndex++;
266
17231691
    buffer->tokenBuffer[ buffer->maxIndex % buffer->bufferSize ] = nextToken(buffer);
267
  }
268
269
91554295
  return buffer->tokenBuffer[ kIndex % buffer->bufferSize ];
270
}
271
272
273
/// As per the normal tokLT but sends information to the debugger
274
///
275
static pANTLR3_COMMON_TOKEN
276
dbgTokLT  (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)
277
{
278
	return tokLT(ts, k);
279
}
280
281
#ifdef	ANTLR3_WINDOWS
282
	/* When fully optimized VC7 complains about non reachable code.
283
	 * Not yet sure if this is an optimizer bug, or a bug in the flow analysis
284
	 */
285
#pragma warning( disable : 4702 )
286
#endif
287
288
static pANTLR3_COMMON_TOKEN
289
get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i)
290
{
291
  Assert(false);  // unimplemented
292
  return NULL;
293
}
294
295
static pANTLR3_TOKEN_SOURCE
296
getTokenSource	(pANTLR3_TOKEN_STREAM ts)
297
{
298
    return  ts->tokenSource;
299
}
300
301
static void
302
setTokenSource	(   pANTLR3_TOKEN_STREAM ts,
303
		    pANTLR3_TOKEN_SOURCE tokenSource)
304
{
305
    ts->tokenSource	= tokenSource;
306
}
307
308
static pANTLR3_STRING
309
toString    (pANTLR3_TOKEN_STREAM ts)
310
{
311
  Assert(false);  // unimplemented
312
  return NULL;
313
}
314
315
static pANTLR3_STRING
316
toStringSS(pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)
317
{
318
  Assert(false);  // unimplemented
319
  return NULL;
320
}
321
322
static pANTLR3_STRING
323
toStringTT  (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop)
324
{
325
  Assert(false);  // unimplemented
326
  return NULL;
327
}
328
329
/** Move the input pointer to the next incoming token.  The stream
330
 *  must become active with LT(1) available.  consume() simply
331
 *  moves the input pointer so that LT(1) points at the next
332
 *  input symbol. Consume at least one token.
333
 *
334
 *  Walk past any token not on the channel the parser is listening to.
335
 */
336
static void
337
17237354
consume	(pANTLR3_INT_STREAM is)
338
{
339
	pANTLR3_COMMON_TOKEN_STREAM cts;
340
	pANTLR3_TOKEN_STREAM	ts;
341
	pBOUNDED_TOKEN_BUFFER buffer;
342
343
17237354
	ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;
344
17237354
	cts	    = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
345
17237354
	buffer      = (pBOUNDED_TOKEN_BUFFER)     cts->super;
346
347
17237354
	buffer->currentIndex++;
348
17237354
}
349
350
351
/// As per ordinary consume but notifies the debugger about hidden
352
/// tokens and so on.
353
///
354
static void
355
dbgConsume	(pANTLR3_INT_STREAM is)
356
{
357
    consume(is);
358
}
359
360
static ANTLR3_UINT32
361
70325562
_LA  (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)
362
{
363
	pANTLR3_TOKEN_STREAM    ts;
364
	pANTLR3_COMMON_TOKEN    tok;
365
366
70325562
	ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;
367
368
70325562
	tok	    =  ts->_LT(ts, i);
369
370
70325550
	if	(tok != NULL)
371
	{
372
70325550
		return	tok->getType(tok);
373
	}
374
	else
375
	{
376
		return	ANTLR3_TOKEN_INVALID;
377
	}
378
}
379
380
/// As per _LA() but for debug mode.
381
///
382
static ANTLR3_UINT32
383
dbgLA  (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)
384
{
385
  Assert(false);
386
  return 0;
387
}
388
389
static ANTLR3_MARKER
390
mark	(pANTLR3_INT_STREAM is)
391
{
392
  is->lastMarker = is->index(is);
393
  return  is->lastMarker;
394
}
395
396
/// As per mark() but with a call to tell the debugger we are doing this
397
///
398
static ANTLR3_MARKER
399
dbgMark	(pANTLR3_INT_STREAM is)
400
{
401
  Assert(false);
402
  return 0;
403
}
404
405
static void
406
release	(pANTLR3_INT_STREAM is, ANTLR3_MARKER mark)
407
{
408
    return;
409
}
410
411
static ANTLR3_UINT32
412
size	(pANTLR3_INT_STREAM is)
413
{
414
  Assert(false);
415
  return 0;
416
}
417
418
static ANTLR3_MARKER
419
52
tindex	(pANTLR3_INT_STREAM is)
420
{
421
  pANTLR3_COMMON_TOKEN_STREAM cts;
422
  pANTLR3_TOKEN_STREAM        ts;
423
  pBOUNDED_TOKEN_BUFFER buffer;
424
425
52
  ts      = (pANTLR3_TOKEN_STREAM)        is->super;
426
52
  cts     = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
427
52
  buffer      = (pBOUNDED_TOKEN_BUFFER)     cts->super;
428
429
52
  return  buffer->currentIndex;
430
}
431
432
static void dbgRewindLast(pANTLR3_INT_STREAM is) { Assert(false); }
433
static void rewindLast(pANTLR3_INT_STREAM is) { Assert(false); }
434
static void
435
rewindStream	(pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)
436
{
437
    is->seek(is, (ANTLR3_UINT32)(marker));
438
}
439
static void
440
dbgRewindStream	(pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)
441
{
442
  Assert(false);
443
}
444
445
static void
446
seek	(pANTLR3_INT_STREAM is, ANTLR3_MARKER index)
447
{
448
    pANTLR3_COMMON_TOKEN_STREAM cts;
449
    pANTLR3_TOKEN_STREAM        ts;
450
451
    ts      = (pANTLR3_TOKEN_STREAM)        is->super;
452
    cts     = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
453
454
    cts->p  = (ANTLR3_UINT32)index;
455
}
456
static void
457
dbgSeek	(pANTLR3_INT_STREAM is, ANTLR3_MARKER index)
458
{
459
  Assert(false);
460
}
461
462
17237443
static pANTLR3_COMMON_TOKEN nextToken(pBOUNDED_TOKEN_BUFFER buffer) {
463
  pANTLR3_COMMON_TOKEN_STREAM tokenStream;
464
  pANTLR3_COMMON_TOKEN tok;
465
  ANTLR3_BOOLEAN discard;
466
467
17237443
  tokenStream = buffer->commonTstream;
468
469
  /*
470
  if( buffer->done == ANTLR3_TRUE ) {
471
    return &(tokenStream->tstream->tokenSource->eofToken);
472
  }
473
  */
474
475
  /* Pick out the next token from the token source
476
   * Remember we just get a pointer (reference if you like) here
477
   * and so if we store it anywhere, we don't set any pointers to auto free it.
478
   */
479
  do {
480
    tok
481
34474886
        = tokenStream->tstream->tokenSource->nextToken(
482
17237443
                                                       tokenStream->tstream->tokenSource);
483
17237431
    if(tok == NULL || tok->type == ANTLR3_TOKEN_EOF) {
484
4932
      buffer->done = ANTLR3_TRUE;
485
4932
      break;
486
    }
487
488
17232499
    discard = ANTLR3_FALSE; /* Assume we are not discarding	*/
489
490
    /* I employ a bit of a trick, or perhaps hack here. Rather than
491
     * store a pointer to a structure in the override map and discard set
492
     * we store the value + 1 cast to a void *. Hence on systems where NULL = (void *)0
493
     * we can distinguish "not being there" from "being channel or type 0"
494
     */
495
496
34464998
    if(tokenStream->discardSet != NULL
497
17232499
        && tokenStream->discardSet->get(tokenStream->discardSet,
498
                                        tok->getType(tok)) != NULL) {
499
      discard = ANTLR3_TRUE;
500
17232499
    } else if(tok->getChannel(tok) != tokenStream->channel) {
501
      discard = ANTLR3_TRUE;
502
    }
503
504
17232499
  } while(discard == ANTLR3_TRUE);
505
506
17237431
  return tok;
507
}
508
509
510
/// Return a string that represents the name associated with the input source
511
///
512
/// /param[in] is The ANTLR3_INT_STREAM interface that is representing this token stream.
513
///
514
/// /returns
515
/// /implements ANTLR3_INT_STREAM_struct::getSourceName()
516
///
517
static pANTLR3_STRING
518
getSourceName				(pANTLR3_INT_STREAM is)
519
{
520
	// Slightly convoluted as we must trace back to the lexer's input source
521
	// via the token source. The streamName that is here is not initialized
522
	// because this is a token stream, not a file or string stream, which are the
523
	// only things that have a context for a source name.
524
	//
525
	return ((pANTLR3_TOKEN_STREAM)(is->super))->tokenSource->fileName;
526
}
527
528
529
}/* CVC4::parser namespace */
530
}/* CVC4 namespace */