C/src/antlr3string.c

/** \file
 * Implementation of the ANTLR3 string and string factory classes
 */

// [The "BSD licence"]
// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC
// http://www.temporal-wave.com
// http://www.linkedin.com/in/jimidle
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
// 3. The name of the author may not be used to endorse or promote products
//    derived from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include    <antlr3string.h>

/* Factory API
 */
static    pANTLR3_STRING    newRaw8	(pANTLR3_STRING_FACTORY factory);
static    pANTLR3_STRING    newRawUTF16	(pANTLR3_STRING_FACTORY factory);
static    pANTLR3_STRING    newSize8	(pANTLR3_STRING_FACTORY factory, ANTLR3_UINT32 size);
static    pANTLR3_STRING    newSizeUTF16	(pANTLR3_STRING_FACTORY factory, ANTLR3_UINT32 size);
static    pANTLR3_STRING    newPtr8	(pANTLR3_STRING_FACTORY factory, pANTLR3_UINT8 string, ANTLR3_UINT32 size);
static    pANTLR3_STRING    newPtrUTF16_8	(pANTLR3_STRING_FACTORY factory, pANTLR3_UINT8 string, ANTLR3_UINT32 size);
static    pANTLR3_STRING    newPtrUTF16_UTF16	(pANTLR3_STRING_FACTORY factory, pANTLR3_UINT8 string, ANTLR3_UINT32 size);
static    pANTLR3_STRING    newStr8	(pANTLR3_STRING_FACTORY factory, pANTLR3_UINT8 string);
static    pANTLR3_STRING    newStrUTF16_8	(pANTLR3_STRING_FACTORY factory, pANTLR3_UINT8 string);
static    pANTLR3_STRING    newStrUTF16_UTF16	(pANTLR3_STRING_FACTORY factory, pANTLR3_UINT8 string);
static    void		    destroy	(pANTLR3_STRING_FACTORY factory, pANTLR3_STRING string);
static    pANTLR3_STRING    printable8	(pANTLR3_STRING_FACTORY factory, pANTLR3_STRING string);
static    pANTLR3_STRING    printableUTF16	(pANTLR3_STRING_FACTORY factory, pANTLR3_STRING string);
static    void		    closeFactory(pANTLR3_STRING_FACTORY factory);

/* String API
 */
static    pANTLR3_UINT8	    set8	(pANTLR3_STRING string, const char * chars);
static    pANTLR3_UINT8	    setUTF16_8	(pANTLR3_STRING string, const char * chars);
static    pANTLR3_UINT8	    setUTF16_UTF16	(pANTLR3_STRING string, const char * chars);
static    pANTLR3_UINT8	    append8	(pANTLR3_STRING string, const char * newbit);
static    pANTLR3_UINT8	    appendUTF16_8	(pANTLR3_STRING string, const char * newbit);
static    pANTLR3_UINT8	    appendUTF16_UTF16	(pANTLR3_STRING string, const char * newbit);
static	  pANTLR3_UINT8	    insert8	(pANTLR3_STRING string, ANTLR3_UINT32 point, const char * newbit);
static	  pANTLR3_UINT8	    insertUTF16_8	(pANTLR3_STRING string, ANTLR3_UINT32 point, const char * newbit);
static	  pANTLR3_UINT8	    insertUTF16_UTF16	(pANTLR3_STRING string, ANTLR3_UINT32 point, const char * newbit);

static    pANTLR3_UINT8	    setS	(pANTLR3_STRING string, pANTLR3_STRING chars);
static    pANTLR3_UINT8	    appendS	(pANTLR3_STRING string, pANTLR3_STRING newbit);
static	  pANTLR3_UINT8	    insertS	(pANTLR3_STRING string, ANTLR3_UINT32 point, pANTLR3_STRING newbit);

static    pANTLR3_UINT8	    addc8	(pANTLR3_STRING string, ANTLR3_UINT32 c);
static    pANTLR3_UINT8	    addcUTF16	(pANTLR3_STRING string, ANTLR3_UINT32 c);
static    pANTLR3_UINT8	    addi8	(pANTLR3_STRING string, ANTLR3_INT32 i);
static    pANTLR3_UINT8	    addiUTF16	(pANTLR3_STRING string, ANTLR3_INT32 i);
static	  pANTLR3_UINT8	    inserti8	(pANTLR3_STRING string, ANTLR3_UINT32 point, ANTLR3_INT32 i);
static	  pANTLR3_UINT8	    insertiUTF16	(pANTLR3_STRING string, ANTLR3_UINT32 point, ANTLR3_INT32 i);

static    ANTLR3_UINT32     compare8	(pANTLR3_STRING string, const char * compStr);
static    ANTLR3_UINT32     compareUTF16_8	(pANTLR3_STRING string, const char * compStr);
static    ANTLR3_UINT32     compareUTF16_UTF16(pANTLR3_STRING string, const char * compStr);
static    ANTLR3_UINT32     compareS	(pANTLR3_STRING string, pANTLR3_STRING compStr);
static    ANTLR3_UCHAR      charAt8	(pANTLR3_STRING string, ANTLR3_UINT32 offset);
static    ANTLR3_UCHAR      charAtUTF16	(pANTLR3_STRING string, ANTLR3_UINT32 offset);
static    pANTLR3_STRING    subString8	(pANTLR3_STRING string, ANTLR3_UINT32 startIndex, ANTLR3_UINT32 endIndex);
static    pANTLR3_STRING    subStringUTF16	(pANTLR3_STRING string, ANTLR3_UINT32 startIndex, ANTLR3_UINT32 endIndex);
static	  ANTLR3_INT32	    toInt32_8	(pANTLR3_STRING string);
static	  ANTLR3_INT32	    toInt32_UTF16  (pANTLR3_STRING string);
static	  pANTLR3_STRING    to8_8		(pANTLR3_STRING string);
static	  pANTLR3_STRING    to8_UTF16		(pANTLR3_STRING string);
static	pANTLR3_STRING		toUTF8_8	(pANTLR3_STRING string);
static	pANTLR3_STRING		toUTF8_UTF16	(pANTLR3_STRING string);

/* Local helpers
 */
static	void			stringInit8	(pANTLR3_STRING string);
static	void			stringInitUTF16	(pANTLR3_STRING string);
static	void	ANTLR3_CDECL	stringFree	(pANTLR3_STRING string);

ANTLR3_API pANTLR3_STRING_FACTORY
antlr3StringFactoryNew(ANTLR3_UINT32 encoding)
{
	pANTLR3_STRING_FACTORY  factory;

	/* Allocate memory
	*/
	factory	= (pANTLR3_STRING_FACTORY) ANTLR3_CALLOC(1, sizeof(ANTLR3_STRING_FACTORY));

	if	(factory == NULL)
	{
		return	NULL;
	}

	/* Now we make a new list to track the strings.
	*/
	factory->strings	= antlr3VectorNew(0);
	factory->index	= 0;

	if	(factory->strings == NULL)
	{
		ANTLR3_FREE(factory);
		return	NULL;
	}

    // Install the API
    //
    // TODO: These encodings need equivalent functions to
    // UTF16 and 8Bit if I am going to support those encodings in the STRING stuff.
	// The STRING stuff was intended as a quick and dirty hack for people that did not
	// want to worry about memory and performance very much, but nobody ever reads the
	// notes or comments or uses the email list search. I want to discourage using these
	// interfaces as it is much more efficient to use the pointers within the tokens
	// directly, so I am not implementing the string stuff for the newer encodings.
    // We install the standard 8 and 16 bit functions for the UTF 8 and 16 but they
	// will not be useful beyond returning the text.
	//
    switch(encoding)
    {
		case    ANTLR3_ENC_UTF32:
			break;

		case    ANTLR3_ENC_UTF32BE:
			break;

		case    ANTLR3_ENC_UTF32LE:
			break;

		case    ANTLR3_ENC_UTF16BE:
		case    ANTLR3_ENC_UTF16LE:
		case    ANTLR3_ENC_UTF16:

			factory->newRaw	    =  newRawUTF16;
			factory->newSize	=  newSizeUTF16;
			factory->newPtr	    =  newPtrUTF16_UTF16;
			factory->newPtr8	=  newPtrUTF16_8;
			factory->newStr	    =  newStrUTF16_UTF16;
			factory->newStr8	=  newStrUTF16_8;
			factory->printable	=  printableUTF16;
			factory->destroy	=  destroy;
			factory->close	    =  closeFactory;
			break;

		case    ANTLR3_ENC_UTF8:
		case    ANTLR3_ENC_EBCDIC:
		case    ANTLR3_ENC_8BIT:
		default:

			factory->newRaw	    =  newRaw8;
			factory->newSize	=  newSize8;
			factory->newPtr	    =  newPtr8;
			factory->newPtr8	=  newPtr8;
			factory->newStr	    =  newStr8;
			factory->newStr8	=  newStr8;
			factory->printable	=  printable8;
			factory->destroy	=  destroy;
			factory->close	    =  closeFactory;
			break;
    }
	return  factory;
}


/**
 *
 * \param factory
 * \return
 */
static    pANTLR3_STRING
newRaw8	(pANTLR3_STRING_FACTORY factory)
{
    pANTLR3_STRING  string;

    string  = (pANTLR3_STRING) ANTLR3_MALLOC(sizeof(ANTLR3_STRING));

    if	(string == NULL)
    {
		return	NULL;
    }

    /* Structure is allocated, now fill in the API etc.
     */
    stringInit8(string);
    string->factory = factory;

    /* Add the string into the allocated list
     */
    factory->strings->set(factory->strings, factory->index, (void *) string, (void (ANTLR3_CDECL *)(void *))(stringFree), ANTLR3_TRUE);
    string->index   = factory->index++;

    return string;
}
/**
 *
 * \param factory
 * \return
 */
static    pANTLR3_STRING
newRawUTF16	(pANTLR3_STRING_FACTORY factory)
{
    pANTLR3_STRING  string;

    string  = (pANTLR3_STRING) ANTLR3_MALLOC(sizeof(ANTLR3_STRING));

    if	(string == NULL)
    {
		return	NULL;
    }

    /* Structure is allocated, now fill in the API etc.
     */
    stringInitUTF16(string);
    string->factory = factory;

    /* Add the string into the allocated list
     */
    factory->strings->set(factory->strings, factory->index, (void *) string, (void (ANTLR3_CDECL *)(void *))(stringFree), ANTLR3_TRUE);
    string->index   = factory->index++;

    return string;
}
static
void	ANTLR3_CDECL stringFree  (pANTLR3_STRING string)
{
    /* First free the string itself if there was anything in it
     */
    if	(string->chars)
    {
	ANTLR3_FREE(string->chars);
    }

    /* Now free the space for this string
     */
    ANTLR3_FREE(string);

    return;
}
/**
 *
 * \param string
 * \return
 */
static	void
stringInit8  (pANTLR3_STRING string)
{
    string->len			= 0;
    string->size		= 0;
    string->chars		= NULL;
    string->encoding	= ANTLR3_ENC_8BIT ;

    /* API for 8 bit strings*/

    string->set		= set8;
    string->set8	= set8;
    string->append	= append8;
    string->append8	= append8;
    string->insert	= insert8;
    string->insert8	= insert8;
    string->addi	= addi8;
    string->inserti	= inserti8;
    string->addc	= addc8;
    string->charAt	= charAt8;
    string->compare	= compare8;
    string->compare8	= compare8;
    string->subString	= subString8;
    string->toInt32	= toInt32_8;
    string->to8		= to8_8;
    string->toUTF8	= toUTF8_8;
    string->compareS	= compareS;
    string->setS	= setS;
    string->appendS	= appendS;
    string->insertS	= insertS;

}
/**
 *
 * \param string
 * \return
 */
static	void
stringInitUTF16  (pANTLR3_STRING string)
{
    string->len		= 0;
    string->size	= 0;
    string->chars	= NULL;
    string->encoding	= ANTLR3_ENC_8BIT;

    /* API for UTF16 strings */

    string->set		= setUTF16_UTF16;
    string->set8	= setUTF16_8;
    string->append	= appendUTF16_UTF16;
    string->append8	= appendUTF16_8;
    string->insert	= insertUTF16_UTF16;
    string->insert8	= insertUTF16_8;
    string->addi	= addiUTF16;
    string->inserti	= insertiUTF16;
    string->addc	= addcUTF16;
    string->charAt	= charAtUTF16;
    string->compare	= compareUTF16_UTF16;
    string->compare8	= compareUTF16_8;
    string->subString	= subStringUTF16;
    string->toInt32	= toInt32_UTF16;
    string->to8		= to8_UTF16;
    string->toUTF8	= toUTF8_UTF16;

    string->compareS	= compareS;
    string->setS	= setS;
    string->appendS	= appendS;
    string->insertS	= insertS;
}
/**
 *
 * \param string
 * \return
 * TODO: Implement UTF-8
 */
static	void
stringInitUTF8  (pANTLR3_STRING string)
{
    string->len	    = 0;
    string->size    = 0;
    string->chars   = NULL;

    /* API */

}

// Convert an 8 bit string into a UTF8 representation, which is in fact just the string itself
// a memcpy as we make no assumptions about the 8 bit encoding.
//
static	pANTLR3_STRING
toUTF8_8	(pANTLR3_STRING string)
{
	return string->factory->newPtr(string->factory, (pANTLR3_UINT8)(string->chars), string->len);
}

// Convert a UTF16 string into a UTF8 representation using the Unicode.org
// supplied C algorithms, which are now contained within the ANTLR3 C runtime
// as permitted by the Unicode license (within the source code antlr3convertutf.c/.h
// UCS2 has the same encoding as UTF16 so we can use UTF16 converter.
//
static	pANTLR3_STRING
toUTF8_UTF16	(pANTLR3_STRING string)
{

    UTF8	      * outputEnd;
    UTF16	      * inputEnd;
    pANTLR3_STRING	utf8String;

    ConversionResult	cResult;

    // Allocate the output buffer, which needs to accommodate potentially
    // 3X (in bytes) the input size (in chars).
    //
    utf8String	= string->factory->newStr8(string->factory, (pANTLR3_UINT8)"");

    if	(utf8String != NULL)
    {
        // Free existing allocation
        //
        ANTLR3_FREE(utf8String->chars);

        // Reallocate according to maximum expected size
        //
        utf8String->size	= string->len *3;
        utf8String->chars	= (pANTLR3_UINT8)ANTLR3_MALLOC(utf8String->size +1);

        if	(utf8String->chars != NULL)
        {
            inputEnd  = (UTF16 *)	(string->chars);
            outputEnd = (UTF8 *)	(utf8String->chars);

            // Call the Unicode converter
            //
            cResult =  ConvertUTF16toUTF8
                (
                (const UTF16**)&inputEnd,
                ((const UTF16 *)(string->chars)) + string->len,
                &outputEnd,
                outputEnd + utf8String->size - 1,
                lenientConversion
                );

            // We don't really care if things failed or not here, we just converted
            // everything that was vaguely possible and stopped when it wasn't. It is
            // up to the grammar programmer to verify that the input is sensible.
            //
            utf8String->len = ANTLR3_UINT32_CAST(((pANTLR3_UINT8)outputEnd) - utf8String->chars);

            *(outputEnd+1) = '\0';		// Always null terminate
        }
    }
    return utf8String;
}

/**
 * Creates a new string with enough capacity for size 8 bit characters plus a terminator.
 *
 * \param[in] factory - Pointer to the string factory that owns strings
 * \param[in] size - In characters
 * \return pointer to the new string.
 */
static    pANTLR3_STRING
newSize8	(pANTLR3_STRING_FACTORY factory, ANTLR3_UINT32 size)
{
    pANTLR3_STRING  string;

    string  = factory->newRaw(factory);

    if	(string == NULL)
    {
        return	string;
    }

    /* Always add one more byte for a terminator ;-)
    */
    string->chars	= (pANTLR3_UINT8) ANTLR3_MALLOC((size_t)(sizeof(ANTLR3_UINT8) * (size+1)));
	if (string->chars != NULL)
    {
		*(string->chars)	= '\0';
		string->size	= size + 1;
	}

    return string;
}
/**
 * Creates a new string with enough capacity for size UTF16 characters plus a terminator.
 *
 * \param[in] factory - Pointer to the string factory that owns strings
 * \param[in] size - In characters (count double for surrogate pairs!!!)
 * \return pointer to the new string.
 */
static    pANTLR3_STRING
newSizeUTF16	(pANTLR3_STRING_FACTORY factory, ANTLR3_UINT32 size)
{
    pANTLR3_STRING  string;

    string  = factory->newRaw(factory);

    if	(string == NULL)
    {
        return	string;
    }

    /* Always add one more byte for a terminator ;-)
    */
    string->chars	= (pANTLR3_UINT8) ANTLR3_MALLOC((size_t)(sizeof(ANTLR3_UINT16) * (size+1)));
    if (string->chars != NULL)
	{
		*(string->chars)	= '\0';
		string->size	= size+1;	/* Size is always in characters, as is len */
	}

    return string;
}

/** Creates a new 8 bit string initialized with the 8 bit characters at the
 *  supplied ptr, of pre-determined size.
 * \param[in] factory - Pointer to the string factory that owns the strings
 * \param[in] ptr - Pointer to 8 bit encoded characters
 * \return pointer to the new string
 */
static    pANTLR3_STRING
newPtr8	(pANTLR3_STRING_FACTORY factory, pANTLR3_UINT8 ptr, ANTLR3_UINT32 size)
{
	pANTLR3_STRING  string;

	string  = factory->newSize(factory, size);

	if	(string == NULL)
	{
		return	NULL;
	}

	if	(size <= 0)
	{
		return	string;
	}

	if	(ptr != NULL)
	{
		ANTLR3_MEMMOVE(string->chars, (const void *)ptr, size);
		*(string->chars + size) = '\0';	    /* Terminate, these strings are usually used for Token streams and printing etc.	*/
		string->len = size;
	}

	return  string;
}

/** Creates a new UTF16 string initialized with the 8 bit characters at the
 *  supplied 8 bit character ptr, of pre-determined size.
 * \param[in] factory - Pointer to the string factory that owns the strings
 * \param[in] ptr - Pointer to 8 bit encoded characters
 * \return pointer to the new string
 */
static    pANTLR3_STRING
newPtrUTF16_8	(pANTLR3_STRING_FACTORY factory, pANTLR3_UINT8 ptr, ANTLR3_UINT32 size)
{
	pANTLR3_STRING  string;

	/* newSize accepts size in characters, not bytes
	*/
	string  = factory->newSize(factory, size);

	if	(string == NULL)
	{
		return	NULL;
	}

	if	(size <= 0)
	{
		return	string;
	}

	if	(ptr != NULL)
	{
		pANTLR3_UINT16	out;
		ANTLR3_INT32    inSize;

		out = (pANTLR3_UINT16)(string->chars);
		inSize	= size;

		while	(inSize-- > 0)
		{
			*out++ = (ANTLR3_UINT16)(*ptr++);
		}

		/* Terminate, these strings are usually used for Token streams and printing etc.
		*/
		*(((pANTLR3_UINT16)(string->chars)) + size) = '\0';

		string->len = size;
	}

	return  string;
}

/** Creates a new UTF16 string initialized with the UTF16 characters at the
 *  supplied ptr, of pre-determined size.
 * \param[in] factory - Pointer to the string factory that owns the strings
 * \param[in] ptr - Pointer to UTF16 encoded characters
 * \return pointer to the new string
 */
static    pANTLR3_STRING
newPtrUTF16_UTF16	(pANTLR3_STRING_FACTORY factory, pANTLR3_UINT8 ptr, ANTLR3_UINT32 size)
{
	pANTLR3_STRING  string;

	string  = factory->newSize(factory, size);

	if	(string == NULL)
	{
		return	NULL;
	}

	if	(size <= 0)
	{
		return	string;
	}

	if	(ptr != NULL)
	{
		ANTLR3_MEMMOVE(string->chars, (const void *)ptr, (size * sizeof(ANTLR3_UINT16)));

		/* Terminate, these strings are usually used for Token streams and printing etc.
		*/
		*(((pANTLR3_UINT16)(string->chars)) + size) = '\0';
		string->len = size;
	}

	return  string;
}

/** Create a new 8 bit string from the supplied, null terminated, 8 bit string pointer.
 * \param[in] factory - Pointer to the string factory that owns strings.
 * \param[in] ptr - Pointer to the 8 bit encoded string
 * \return Pointer to the newly initialized string
 */
static    pANTLR3_STRING
newStr8	(pANTLR3_STRING_FACTORY factory, pANTLR3_UINT8 ptr)
{
    return factory->newPtr8(factory, ptr, (ANTLR3_UINT32)strlen((const char *)ptr));
}

/** Create a new UTF16 string from the supplied, null terminated, 8 bit string pointer.
 * \param[in] factory - Pointer to the string factory that owns strings.
 * \param[in] ptr - Pointer to the 8 bit encoded string
 * \return POinter to the newly initialized string
 */
static    pANTLR3_STRING
newStrUTF16_8	(pANTLR3_STRING_FACTORY factory, pANTLR3_UINT8 ptr)
{
    return factory->newPtr8(factory, ptr, (ANTLR3_UINT32)strlen((const char *)ptr));
}

/** Create a new UTF16 string from the supplied, null terminated, UTF16 string pointer.
 * \param[in] factory - Pointer to the string factory that owns strings.
 * \param[in] ptr - Pointer to the UTF16 encoded string
 * \return Pointer to the newly initialized string
 */
static    pANTLR3_STRING
newStrUTF16_UTF16	(pANTLR3_STRING_FACTORY factory, pANTLR3_UINT8 ptr)
{
    pANTLR3_UINT16  in;
    ANTLR3_UINT32   count;

    /** First, determine the length of the input string
     */
    in	    = (pANTLR3_UINT16)ptr;
    count   = 0;

    while   (*in++ != '\0')
    {
		count++;
    }
    return factory->newPtr(factory, ptr, count);
}

static    void
destroy	(pANTLR3_STRING_FACTORY factory, pANTLR3_STRING string)
{
    // Record which string we are deleting
    //
    ANTLR3_UINT32 strIndex = string->index;

    // Ensure that the string was not factory made, or we would try
    // to delete memory that wasn't allocated outside the factory
    // block.
    // Remove the specific indexed string from the vector
    //
    factory->strings->del(factory->strings, strIndex);

    // One less string in the vector, so decrement the factory index
    // so that the next string allocated is indexed correctly with
    // respect to the vector.
    //
    factory->index--;

    // Now we have to reindex the strings in the vector that followed
    // the one we just deleted. We only do this if the one we just deleted
    // was not the last one.
    //
    if  (strIndex< factory->index)
    {
        // We must reindex the strings after the one we just deleted.
        // The one that follows the one we just deleted is also out
        // of whack, so we start there.
        //
        ANTLR3_UINT32 i;

        for (i = strIndex; i < factory->index; i++)
        {
            // Renumber the entry
            //
            ((pANTLR3_STRING)(factory->strings->elements[i].element))->index = i;
        }
    }

    // The string has been destroyed and the elements of the factory are reindexed.
    //

}

static    pANTLR3_STRING
printable8(pANTLR3_STRING_FACTORY factory, pANTLR3_STRING instr)
{
    pANTLR3_STRING  string;

    /* We don't need to be too efficient here, this is mostly for error messages and so on.
     */
    pANTLR3_UINT8   scannedText;
    ANTLR3_UINT32   i;

    /* Assume we need as much as twice as much space to parse out the control characters
     */
    string  = factory->newSize(factory, instr->len *2 + 1);

    /* Scan through and replace unprintable (in terms of this routine)
     * characters
     */
    scannedText = string->chars;

    for	(i = 0; i < instr->len; i++)
    {
		if (*(instr->chars + i) == '\n')
		{
			*scannedText++ = '\\';
			*scannedText++ = 'n';
		}
		else if (*(instr->chars + i) == '\r')
		{
			*scannedText++ = '\\';
			*scannedText++ = 'r';
		}
		else if	(!isprint(*(instr->chars +i)))
		{
			*scannedText++ = '?';
		}
		else
		{
			*scannedText++ = *(instr->chars + i);
		}
    }
    *scannedText  = '\0';

    string->len	= (ANTLR3_UINT32)(scannedText - string->chars);

    return  string;
}

static    pANTLR3_STRING
printableUTF16(pANTLR3_STRING_FACTORY factory, pANTLR3_STRING instr)
{
    pANTLR3_STRING  string;

    /* We don't need to be too efficient here, this is mostly for error messages and so on.
     */
    pANTLR3_UINT16  scannedText;
    pANTLR3_UINT16  inText;
    ANTLR3_UINT32   i;
    ANTLR3_UINT32   outLen;

    /* Assume we need as much as twice as much space to parse out the control characters
     */
    string  = factory->newSize(factory, instr->len *2 + 1);

    /* Scan through and replace unprintable (in terms of this routine)
     * characters
     */
    scannedText = (pANTLR3_UINT16)(string->chars);
    inText	= (pANTLR3_UINT16)(instr->chars);
    outLen	= 0;

    for	(i = 0; i < instr->len; i++)
    {
		if (*(inText + i) == '\n')
		{
			*scannedText++   = '\\';
			*scannedText++   = 'n';
			outLen	    += 2;
		}
		else if (*(inText + i) == '\r')
		{
			*scannedText++   = '\\';
			*scannedText++   = 'r';
			outLen	    += 2;
		}
		else if	(!isprint(*(inText +i)))
		{
			*scannedText++ = '?';
			outLen++;
		}
		else
		{
			*scannedText++ = *(inText + i);
			outLen++;
		}
    }
    *scannedText  = '\0';

    string->len	= outLen;

    return  string;
}

/** Fascist Capitalist Pig function created
 *  to oppress the workers comrade.
 */
static    void
closeFactory	(pANTLR3_STRING_FACTORY factory)
{
    /* Delete the vector we were tracking the strings with, this will
     * causes all the allocated strings to be deallocated too
     */
    factory->strings->free(factory->strings);

    /* Delete the space for the factory itself
     */
    ANTLR3_FREE((void *)factory);
}

static    pANTLR3_UINT8
append8	(pANTLR3_STRING string, const char * newbit)
{
    ANTLR3_UINT32 len;

    len	= (ANTLR3_UINT32)strlen(newbit);

    if	(string->size < (string->len + len + 1))
    {
		pANTLR3_UINT8 newAlloc = (pANTLR3_UINT8) ANTLR3_REALLOC((void *)string->chars, (ANTLR3_UINT32)(string->len + len + 1));
		if (newAlloc == NULL)
		{
			return NULL;
		}
		string->chars	= newAlloc;
		string->size	= string->len + len + 1;
    }

    /* Note we copy one more byte than the strlen in order to get the trailing
     */
    ANTLR3_MEMMOVE((void *)(string->chars + string->len), newbit, (ANTLR3_UINT32)(len+1));
    string->len	+= len;

    return string->chars;
}

static    pANTLR3_UINT8
appendUTF16_8	(pANTLR3_STRING string, const char * newbit)
{
    ANTLR3_UINT32   len;
    pANTLR3_UINT16  apPoint;
    ANTLR3_UINT32   count;

    len	= (ANTLR3_UINT32)strlen(newbit);

    if	(string->size < (string->len + len + 1))
    {
		pANTLR3_UINT8 newAlloc = (pANTLR3_UINT8) ANTLR3_REALLOC((void *)string->chars, (ANTLR3_UINT32)((sizeof(ANTLR3_UINT16)*(string->len + len + 1))));
		if (newAlloc == NULL)
		{
			return NULL;
		}
		string->chars	= newAlloc;
		string->size	= string->len + len + 1;
    }

    apPoint = ((pANTLR3_UINT16)string->chars) + string->len;
    string->len	+= len;

    for	(count = 0; count < len; count++)
    {
		*apPoint++   = *(newbit + count);
    }
    *apPoint = '\0';

    return string->chars;
}

static    pANTLR3_UINT8
appendUTF16_UTF16	(pANTLR3_STRING string, const char * newbit)
{
    ANTLR3_UINT32 len;
    pANTLR3_UINT16  in;

    /** First, determine the length of the input string
     */
    in	    = (pANTLR3_UINT16)newbit;
    len   = 0;

    while   (*in++ != '\0')
    {
		len++;
    }

    if	(string->size < (string->len + len + 1))
    {
		pANTLR3_UINT8 newAlloc = (pANTLR3_UINT8) ANTLR3_REALLOC((void *)string->chars, (ANTLR3_UINT32)( sizeof(ANTLR3_UINT16) *(string->len + len + 1) ));
		if (newAlloc == NULL)
		{
			return NULL;
		}
		string->chars	= newAlloc;
		string->size	= string->len + len + 1;
    }

    /* Note we copy one more byte than the strlen in order to get the trailing delimiter
     */
    ANTLR3_MEMMOVE((void *)(((pANTLR3_UINT16)string->chars) + string->len), newbit, (ANTLR3_UINT32)(sizeof(ANTLR3_UINT16)*(len+1)));
    string->len	+= len;

    return string->chars;
}

static    pANTLR3_UINT8
set8	(pANTLR3_STRING string, const char * chars)
{
    ANTLR3_UINT32	len;

    len = (ANTLR3_UINT32)strlen(chars);
    if	(string->size < len + 1)
    {
		pANTLR3_UINT8 newAlloc = (pANTLR3_UINT8) ANTLR3_REALLOC((void *)string->chars, (ANTLR3_UINT32)(len + 1));
		if (newAlloc == NULL)
		{
			return NULL;
		}
		string->chars	= newAlloc;
		string->size	= len + 1;
    }

    /* Note we copy one more byte than the strlen in order to get the trailing '\0'
     */
    ANTLR3_MEMMOVE((void *)(string->chars), chars, (ANTLR3_UINT32)(len+1));
    string->len	    = len;

    return  string->chars;

}

static    pANTLR3_UINT8
setUTF16_8	(pANTLR3_STRING string, const char * chars)
{
    ANTLR3_UINT32	len;
    ANTLR3_UINT32	count;
    pANTLR3_UINT16	apPoint;

    len = (ANTLR3_UINT32)strlen(chars);
    if	(string->size < len + 1)
	{
		pANTLR3_UINT8 newAlloc = (pANTLR3_UINT8) ANTLR3_REALLOC((void *)string->chars, (ANTLR3_UINT32)(sizeof(ANTLR3_UINT16)*(len + 1)));
		if (newAlloc == NULL)
		{
			return NULL;
		}
		string->chars	= newAlloc;
		string->size	= len + 1;
    }
    apPoint = ((pANTLR3_UINT16)string->chars);
    string->len	= len;

    for	(count = 0; count < string->len; count++)
    {
		*apPoint++   = *(chars + count);
    }
    *apPoint = '\0';

    return  string->chars;
}

static    pANTLR3_UINT8
setUTF16_UTF16    (pANTLR3_STRING string, const char * chars)
{
    ANTLR3_UINT32   len;
    pANTLR3_UINT16  in;

    /** First, determine the length of the input string
     */
    in	    = (pANTLR3_UINT16)chars;
    len   = 0;

    while   (*in++ != '\0')
    {
		len++;
    }

    if	(string->size < len + 1)
    {
		pANTLR3_UINT8 newAlloc = (pANTLR3_UINT8) ANTLR3_REALLOC((void *)string->chars, (ANTLR3_UINT32)(sizeof(ANTLR3_UINT16)*(len + 1)));
		if (newAlloc == NULL)
		{
			return NULL;
		}
		string->chars	= newAlloc;
		string->size	= len + 1;
    }

    /* Note we copy one more byte than the strlen in order to get the trailing '\0'
     */
    ANTLR3_MEMMOVE((void *)(string->chars), chars, (ANTLR3_UINT32)((len+1) * sizeof(ANTLR3_UINT16)));
    string->len	    = len;

    return  string->chars;

}

static    pANTLR3_UINT8
addc8	(pANTLR3_STRING string, ANTLR3_UINT32 c)
{
    if	(string->size < string->len + 2)
    {
		pANTLR3_UINT8 newAlloc = (pANTLR3_UINT8) ANTLR3_REALLOC((void *)string->chars, (ANTLR3_UINT32)(string->len + 2));
		if (newAlloc == NULL)
		{
			return NULL;
		}
		string->chars	= newAlloc;
		string->size	= string->len + 2;
    }
    *(string->chars + string->len)	= (ANTLR3_UINT8)c;
    *(string->chars + string->len + 1)	= '\0';
    string->len++;

    return  string->chars;
}

static    pANTLR3_UINT8
addcUTF16	(pANTLR3_STRING string, ANTLR3_UINT32 c)
{
    pANTLR3_UINT16  ptr;

    if	(string->size < string->len + 2)
    {
		pANTLR3_UINT8 newAlloc = (pANTLR3_UINT8) ANTLR3_REALLOC((void *)string->chars, (ANTLR3_UINT32)(sizeof(ANTLR3_UINT16) * (string->len + 2)));
		if (newAlloc == NULL)
		{
			return NULL;
		}
		string->chars	= newAlloc;
		string->size	= string->len + 2;
    }
    ptr	= (pANTLR3_UINT16)(string->chars);

    *(ptr + string->len)	= (ANTLR3_UINT16)c;
    *(ptr + string->len + 1)	= '\0';
    string->len++;

    return  string->chars;
}

static    pANTLR3_UINT8
addi8	(pANTLR3_STRING string, ANTLR3_INT32 i)
{
    ANTLR3_UINT8	    newbit[32];

    sprintf((char *)newbit, "%d", i);

    return  string->append8(string, (const char *)newbit);
}
static    pANTLR3_UINT8
addiUTF16	(pANTLR3_STRING string, ANTLR3_INT32 i)
{
    ANTLR3_UINT8	    newbit[32];

    sprintf((char *)newbit, "%d", i);

    return  string->append8(string, (const char *)newbit);
}

static	  pANTLR3_UINT8
inserti8    (pANTLR3_STRING string, ANTLR3_UINT32 point, ANTLR3_INT32 i)
{
    ANTLR3_UINT8	    newbit[32];

    sprintf((char *)newbit, "%d", i);
    return  string->insert8(string, point, (const char *)newbit);
}
static	  pANTLR3_UINT8
insertiUTF16    (pANTLR3_STRING string, ANTLR3_UINT32 point, ANTLR3_INT32 i)
{
    ANTLR3_UINT8	    newbit[32];

    sprintf((char *)newbit, "%d", i);
    return  string->insert8(string, point, (const char *)newbit);
}

static	pANTLR3_UINT8
insert8	(pANTLR3_STRING string, ANTLR3_UINT32 point, const char * newbit)
{
    ANTLR3_UINT32	len;

    if	(point >= string->len)
    {
		return	string->append(string, newbit);
    }

    len	= (ANTLR3_UINT32)strlen(newbit);

    if	(len == 0)
    {
		return	string->chars;
    }

    if	(string->size < (string->len + len + 1))
    {
		pANTLR3_UINT8 newAlloc = (pANTLR3_UINT8) ANTLR3_REALLOC((void *)string->chars, (ANTLR3_UINT32)(string->len + len + 1));
		if (newAlloc == NULL)
		{
			return NULL;
		}
		string->chars	= newAlloc;
		string->size	= string->len + len + 1;
    }

    /* Move the characters we are inserting before, including the delimiter
     */
    ANTLR3_MEMMOVE((void *)(string->chars + point + len), (void *)(string->chars + point), (ANTLR3_UINT32)(string->len - point + 1));

    /* Note we copy the exact number of bytes
     */
    ANTLR3_MEMMOVE((void *)(string->chars + point), newbit, (ANTLR3_UINT32)(len));

    string->len += len;

    return  string->chars;
}

static	pANTLR3_UINT8
insertUTF16_8	(pANTLR3_STRING string, ANTLR3_UINT32 point, const char * newbit)
{
    ANTLR3_UINT32	len;
    ANTLR3_UINT32	count;
    pANTLR3_UINT16	inPoint;

    if	(point >= string->len)
    {
		return	string->append8(string, newbit);
    }

    len	= (ANTLR3_UINT32)strlen(newbit);

    if	(len == 0)
    {
		return	string->chars;
    }

    if	(string->size < (string->len + len + 1))
    {
		pANTLR3_UINT8 newAlloc = (pANTLR3_UINT8) ANTLR3_REALLOC((void *)string->chars, (ANTLR3_UINT32)(sizeof(ANTLR3_UINT16)*(string->len + len + 1)));
		if (newAlloc == NULL)
		{
			return NULL;
		}
		string->chars	= newAlloc;
		string->size	= string->len + len + 1;
    }

    /* Move the characters we are inserting before, including the delimiter
     */
    ANTLR3_MEMMOVE((void *)(((pANTLR3_UINT16)string->chars) + point + len), (void *)(((pANTLR3_UINT16)string->chars) + point), (ANTLR3_UINT32)(sizeof(ANTLR3_UINT16)*(string->len - point + 1)));

    string->len += len;

    inPoint = ((pANTLR3_UINT16)(string->chars))+point;
    for	(count = 0; count<len; count++)
    {
		*(inPoint + count) = (ANTLR3_UINT16)(*(newbit+count));
    }

    return  string->chars;
}

static	pANTLR3_UINT8
insertUTF16_UTF16	(pANTLR3_STRING string, ANTLR3_UINT32 point, const char * newbit)
{
    ANTLR3_UINT32	len;
    pANTLR3_UINT16	in;

    if	(point >= string->len)
    {
		return	string->append(string, newbit);
    }

    /** First, determine the length of the input string
     */
    in	    = (pANTLR3_UINT16)newbit;
    len	    = 0;

    while   (*in++ != '\0')
    {
		len++;
    }

    if	(len == 0)
    {
		return	string->chars;
    }

    if	(string->size < (string->len + len + 1))
    {
		pANTLR3_UINT8 newAlloc = (pANTLR3_UINT8) ANTLR3_REALLOC((void *)string->chars, (ANTLR3_UINT32)(sizeof(ANTLR3_UINT16)*(string->len + len + 1)));
		if (newAlloc == NULL)
		{
			return NULL;
		}
		string->chars	= newAlloc;
		string->size	= string->len + len + 1;
    }

    /* Move the characters we are inserting before, including the delimiter
     */
    ANTLR3_MEMMOVE((void *)(((pANTLR3_UINT16)string->chars) + point + len), (void *)(((pANTLR3_UINT16)string->chars) + point), (ANTLR3_UINT32)(sizeof(ANTLR3_UINT16)*(string->len - point + 1)));


    /* Note we copy the exact number of characters
     */
    ANTLR3_MEMMOVE((void *)(((pANTLR3_UINT16)string->chars) + point), newbit, (ANTLR3_UINT32)(sizeof(ANTLR3_UINT16)*(len)));

    string->len += len;

    return  string->chars;
}

static    pANTLR3_UINT8	    setS	(pANTLR3_STRING string, pANTLR3_STRING chars)
{
    return  string->set(string, (const char *)(chars->chars));
}

static    pANTLR3_UINT8	    appendS	(pANTLR3_STRING string, pANTLR3_STRING newbit)
{
    /* We may be passed an empty string, in which case we just return the current pointer
     */
    if	(newbit == NULL || newbit->len == 0 || newbit->size == 0 || newbit->chars == NULL)
    {
		return	string->chars;
    }
    else
    {
		return  string->append(string, (const char *)(newbit->chars));
    }
}

static	  pANTLR3_UINT8	    insertS	(pANTLR3_STRING string, ANTLR3_UINT32 point, pANTLR3_STRING newbit)
{
    return  string->insert(string, point, (const char *)(newbit->chars));
}

/* Function that compares the text of a string to the supplied
 * 8 bit character string and returns a result a la strcmp()
 */
static ANTLR3_UINT32
compare8	(pANTLR3_STRING string, const char * compStr)
{
    return  strcmp((const char *)(string->chars), compStr);
}

/* Function that compares the text of a string with the supplied character string
 * (which is assumed to be in the same encoding as the string itself) and returns a result
 * a la strcmp()
 */
static ANTLR3_UINT32
compareUTF16_8	(pANTLR3_STRING string, const char * compStr)
{
    pANTLR3_UINT16  ourString;
    ANTLR3_UINT32   charDiff;

    ourString	= (pANTLR3_UINT16)(string->chars);

    while   (((ANTLR3_UCHAR)(*ourString) != '\0') && ((ANTLR3_UCHAR)(*compStr) != '\0'))
    {
		charDiff = *ourString - *compStr;
		if  (charDiff != 0)
		{
			return charDiff;
		}
		ourString++;
		compStr++;
    }

    /* At this point, one of the strings was terminated
     */
    return (ANTLR3_UINT32)((ANTLR3_UCHAR)(*ourString) - (ANTLR3_UCHAR)(*compStr));

}

/* Function that compares the text of a string with the supplied character string
 * (which is assumed to be in the same encoding as the string itself) and returns a result
 * a la strcmp()
 */
static ANTLR3_UINT32
compareUTF16_UTF16	(pANTLR3_STRING string, const char * compStr8)
{
    pANTLR3_UINT16  ourString;
    pANTLR3_UINT16  compStr;
    ANTLR3_UINT32   charDiff;

    ourString	= (pANTLR3_UINT16)(string->chars);
    compStr	= (pANTLR3_UINT16)(compStr8);

    while   (((ANTLR3_UCHAR)(*ourString) != '\0') && ((ANTLR3_UCHAR)(*((pANTLR3_UINT16)compStr)) != '\0'))
    {
		charDiff = *ourString - *compStr;
		if  (charDiff != 0)
		{
			return charDiff;
		}
		ourString++;
		compStr++;
    }

    /* At this point, one of the strings was terminated
     */
    return (ANTLR3_UINT32)((ANTLR3_UCHAR)(*ourString) - (ANTLR3_UCHAR)(*compStr));
}

/* Function that compares the text of a string with the supplied string
 * (which is assumed to be in the same encoding as the string itself) and returns a result
 * a la strcmp()
 */
static ANTLR3_UINT32
compareS    (pANTLR3_STRING string, pANTLR3_STRING compStr)
{
    return  string->compare(string, (const char *)compStr->chars);
}


/* Function that returns the character indexed at the supplied
 * offset as a 32 bit character.
 */
static ANTLR3_UCHAR
charAt8	    (pANTLR3_STRING string, ANTLR3_UINT32 offset)
{
    if	(offset > string->len)
    {
		return (ANTLR3_UCHAR)'\0';
    }
    else
    {
		return  (ANTLR3_UCHAR)(*(string->chars + offset));
    }
}

/* Function that returns the character indexed at the supplied
 * offset as a 32 bit character.
 */
static ANTLR3_UCHAR
charAtUTF16    (pANTLR3_STRING string, ANTLR3_UINT32 offset)
{
    if	(offset > string->len)
    {
		return (ANTLR3_UCHAR)'\0';
    }
    else
    {
		return  (ANTLR3_UCHAR)(*((pANTLR3_UINT16)(string->chars) + offset));
    }
}

/* Function that returns a substring of the supplied string a la .subString(s,e)
 * in java runtimes.
 */
static pANTLR3_STRING
subString8   (pANTLR3_STRING string, ANTLR3_UINT32 startIndex, ANTLR3_UINT32 endIndex)
{
    pANTLR3_STRING newStr;

    if	(endIndex > string->len)
    {
		endIndex = string->len + 1;
    }
    newStr  = string->factory->newPtr(string->factory, string->chars + startIndex, endIndex - startIndex);

    return newStr;
}

/* Returns a substring of the supplied string a la .subString(s,e)
 * in java runtimes.
 */
static pANTLR3_STRING
subStringUTF16  (pANTLR3_STRING string, ANTLR3_UINT32 startIndex, ANTLR3_UINT32 endIndex)
{
    pANTLR3_STRING newStr;

    if	(endIndex > string->len)
    {
		endIndex = string->len + 1;
    }
    newStr  = string->factory->newPtr(string->factory, (pANTLR3_UINT8)((pANTLR3_UINT16)(string->chars) + startIndex), endIndex - startIndex);

    return newStr;
}

/* Function that can convert the characters in the string to an integer
 */
static ANTLR3_INT32
toInt32_8	    (struct ANTLR3_STRING_struct * string)
{
    return  atoi((const char *)(string->chars));
}

/* Function that can convert the characters in the string to an integer
 */
static ANTLR3_INT32
toInt32_UTF16       (struct ANTLR3_STRING_struct * string)
{
    pANTLR3_UINT16  input;
    ANTLR3_INT32   value;
    ANTLR3_BOOLEAN  negate;

    value   = 0;
    input   = (pANTLR3_UINT16)(string->chars);
    negate  = ANTLR3_FALSE;

    if	(*input == (ANTLR3_UCHAR)'-')
    {
		negate = ANTLR3_TRUE;
		input++;
    }
    else if (*input == (ANTLR3_UCHAR)'+')
    {
		input++;
    }

    while   (*input != '\0' && isdigit(*input))
    {
		value	 = value * 10;
		value	+= ((ANTLR3_UINT32)(*input) - (ANTLR3_UINT32)'0');
		input++;
    }

    return negate ? -value : value;
}

/* Function that returns a pointer to an 8 bit version of the string,
 * which in this case is just the string as this is
 * 8 bit encodiing anyway.
 */
static	  pANTLR3_STRING	    to8_8	(pANTLR3_STRING string)
{
    return  string;
}

/* Function that returns an 8 bit version of the string,
 * which in this case is returning all the UTF16 characters
 * narrowed back into 8 bits, with characters that are too large
 * replaced with '_'
 */
static	  pANTLR3_STRING    to8_UTF16	(pANTLR3_STRING string)
{
	pANTLR3_STRING  newStr;
	ANTLR3_UINT32   i;

	/* Create a new 8 bit string
	*/
	newStr  = newRaw8(string->factory);

	if	(newStr == NULL)
	{
		return	NULL;
	}

	/* Always add one more byte for a terminator
	*/
	newStr->chars   = (pANTLR3_UINT8) ANTLR3_MALLOC((size_t)(string->len + 1));
	if (newStr->chars != NULL)
	{
		newStr->size    = string->len + 1;
		newStr->len	    = string->len;

		/* Now copy each UTF16 charActer , making it an 8 bit character of
		* some sort.
		*/
		for	(i=0; i<string->len; i++)
		{
			ANTLR3_UCHAR	c;

			c = *(((pANTLR3_UINT16)(string->chars)) + i);

			*(newStr->chars + i) = (ANTLR3_UINT8)(c > 255 ? '_' : c);
		}

		/* Terminate
		*/
		*(newStr->chars + newStr->len) = '\0';
	}

	return newStr;
}