tools/gennorm2/norms.cpp

// © 2017 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html

// norms.cpp
// created: 2017jun04 Markus W. Scherer
// (pulled out of n2builder.cpp)

#include "unicode/utypes.h"

#if !UCONFIG_NO_NORMALIZATION

#include <stdio.h>
#include <stdlib.h>
#include "unicode/errorcode.h"
#include "unicode/umutablecptrie.h"
#include "unicode/unistr.h"
#include "unicode/utf16.h"
#include "normalizer2impl.h"
#include "norms.h"
#include "toolutil.h"
#include "uvectr32.h"

U_NAMESPACE_BEGIN

void BuilderReorderingBuffer::append(UChar32 c, uint8_t cc) {
    if(cc==0 || fLength==0 || ccAt(fLength-1)<=cc) {
        if(cc==0) {
            fLastStarterIndex=fLength;
        }
        fArray[fLength++]=(c<<8)|cc;
        return;
    }
    // Let this character bubble back to its canonical order.
    int32_t i=fLength-1;
    while(i>fLastStarterIndex && ccAt(i)>cc) {
        --i;
    }
    ++i;  // after the last starter or prevCC<=cc
    // Move this and the following characters forward one to make space.
    for(int32_t j=fLength; i<j; --j) {
        fArray[j]=fArray[j-1];
    }
    fArray[i]=(c<<8)|cc;
    ++fLength;
    fDidReorder=true;
}

void BuilderReorderingBuffer::toString(UnicodeString &dest) const {
    dest.remove();
    for(int32_t i=0; i<fLength; ++i) {
        dest.append(charAt(i));
    }
}

UChar32 Norm::combine(UChar32 trail) const {
    int32_t length;
    const CompositionPair *pairs=getCompositionPairs(length);
    for(int32_t i=0; i<length; ++i) {
        if(trail==pairs[i].trail) {
            return pairs[i].composite;
        }
        if(trail<pairs[i].trail) {
            break;
        }
    }
    return U_SENTINEL;
}

Norms::Norms(UErrorCode &errorCode) {
    normTrie = umutablecptrie_open(0, 0, &errorCode);
    normMem=utm_open("gennorm2 normalization structs", 10000, 0x110100, sizeof(Norm));
    // Default "inert" Norm struct at index 0. Practically immutable.
    norms=allocNorm();
    norms->type=Norm::INERT;
}

Norms::~Norms() {
    umutablecptrie_close(normTrie);
    int32_t normsLength=utm_countItems(normMem);
    for(int32_t i=1; i<normsLength; ++i) {
        delete norms[i].mapping;
        delete norms[i].rawMapping;
        delete norms[i].compositions;
    }
    utm_close(normMem);
}

Norm *Norms::allocNorm() {
    Norm* p = static_cast<Norm*>(utm_alloc(normMem));
    norms = static_cast<Norm*>(utm_getStart(normMem)); // in case it got reallocated
    return p;
}

Norm *Norms::getNorm(UChar32 c) {
    uint32_t i = umutablecptrie_get(normTrie, c);
    if(i==0) {
        return nullptr;
    }
    return norms+i;
}

const Norm *Norms::getNorm(UChar32 c) const {
    uint32_t i = umutablecptrie_get(normTrie, c);
    if(i==0) {
        return nullptr;
    }
    return norms+i;
}

const Norm &Norms::getNormRef(UChar32 c) const {
    return norms[umutablecptrie_get(normTrie, c)];
}

Norm *Norms::createNorm(UChar32 c) {
    uint32_t i=umutablecptrie_get(normTrie, c);
    if(i!=0) {
        return norms+i;
    } else {
        /* allocate Norm */
        Norm *p=allocNorm();
        IcuToolErrorCode errorCode("gennorm2/createNorm()");
        umutablecptrie_set(normTrie, c, static_cast<uint32_t>(p - norms), errorCode);
        return p;
    }
}

void Norms::reorder(UnicodeString &mapping, BuilderReorderingBuffer &buffer) const {
    int32_t length=mapping.length();
    U_ASSERT(length<=Normalizer2Impl::MAPPING_LENGTH_MASK);
    const char16_t *s=mapping.getBuffer();
    int32_t i=0;
    UChar32 c;
    while(i<length) {
        U16_NEXT(s, i, length, c);
        buffer.append(c, getCC(c));
    }
    if(buffer.didReorder()) {
        buffer.toString(mapping);
    }
}

UBool Norms::combinesWithCCBetween(const Norm &norm, uint8_t lowCC, int32_t highCC) const {
    if((highCC-lowCC)>=2) {
        int32_t length;
        const CompositionPair *pairs=norm.getCompositionPairs(length);
        for(int32_t i=0; i<length; ++i) {
            uint8_t trailCC=getCC(pairs[i].trail);
            if(lowCC<trailCC && trailCC<highCC) {
                return true;
            }
        }
    }
    return false;
}

void Norms::enumRanges(Enumerator &e) {
    UChar32 start = 0, end;
    uint32_t i;
    while ((end = umutablecptrie_getRange(normTrie, start, UCPMAP_RANGE_NORMAL, 0,
                                          nullptr, nullptr, &i)) >= 0) {
        if (i > 0) {
            e.rangeHandler(start, end, norms[i]);
        }
        start = end + 1;
    }
}

Norms::Enumerator::~Enumerator() {}

void CompositionBuilder::rangeHandler(UChar32 start, UChar32 end, Norm &norm) {
    if(norm.mappingType!=Norm::ROUND_TRIP) { return; }
    if(start!=end) {
        fprintf(stderr,
                "gennorm2 error: same round-trip mapping for "
                "more than 1 code point U+%04lX..U+%04lX\n",
                static_cast<long>(start), static_cast<long>(end));
        exit(U_INVALID_FORMAT_ERROR);
    }
    if(norm.cc!=0) {
        fprintf(stderr,
                "gennorm2 error: "
                "U+%04lX has a round-trip mapping and ccc!=0, "
                "not possible in Unicode normalization\n",
                static_cast<long>(start));
        exit(U_INVALID_FORMAT_ERROR);
    }
    // setRoundTripMapping() ensured that there are exactly two code points.
    const UnicodeString &m=*norm.mapping;
    UChar32 lead=m.char32At(0);
    UChar32 trail=m.char32At(m.length()-1);
    if(norms.getCC(lead)!=0) {
        fprintf(stderr,
                "gennorm2 error: "
                "U+%04lX's round-trip mapping's starter U+%04lX has ccc!=0, "
                "not possible in Unicode normalization\n",
                static_cast<long>(start), static_cast<long>(lead));
        exit(U_INVALID_FORMAT_ERROR);
    }
    // Flag for trailing character.
    norms.createNorm(trail)->combinesBack=true;
    // Insert (trail, composite) pair into compositions list for the lead character.
    IcuToolErrorCode errorCode("gennorm2/addComposition()");
    Norm *leadNorm=norms.createNorm(lead);
    UVector32 *compositions=leadNorm->compositions;
    int32_t i;
    if(compositions==nullptr) {
        compositions=leadNorm->compositions=new UVector32(errorCode);
        i=0;  // "insert" the first pair at index 0
    } else {
        // Insertion sort, and check for duplicate trail characters.
        int32_t length;
        const CompositionPair *pairs=leadNorm->getCompositionPairs(length);
        for(i=0; i<length; ++i) {
            if(trail==pairs[i].trail) {
                fprintf(stderr,
                        "gennorm2 error: same round-trip mapping for "
                        "more than 1 code point (e.g., U+%04lX) to U+%04lX + U+%04lX\n",
                        static_cast<long>(start), static_cast<long>(lead), static_cast<long>(trail));
                exit(U_INVALID_FORMAT_ERROR);
            }
            if(trail<pairs[i].trail) {
                break;
            }
        }
    }
    compositions->insertElementAt(trail, 2*i, errorCode);
    compositions->insertElementAt(start, 2*i+1, errorCode);
}

void Decomposer::rangeHandler(UChar32 start, UChar32 end, Norm &norm) {
    if(!norm.hasMapping()) { return; }
    const UnicodeString &m=*norm.mapping;
    UnicodeString *decomposed=nullptr;
    const char16_t *s=toUCharPtr(m.getBuffer());
    int32_t length=m.length();
    int32_t prev, i=0;
    UChar32 c;
    while(i<length) {
        prev=i;
        U16_NEXT(s, i, length, c);
        if(start<=c && c<=end) {
            fprintf(stderr,
                    "gennorm2 error: U+%04lX maps to itself directly or indirectly\n",
                    static_cast<long>(c));
            exit(U_INVALID_FORMAT_ERROR);
        }
        const Norm &cNorm=norms.getNormRef(c);
        if(norm.mappingType==Norm::ROUND_TRIP && prev==0 &&
                !norm.combinesBack && cNorm.combinesBack) {
            // If a two-way mapping starts with an NFC_QC=Maybe character,
            // then mark the composite as NFC_QC=Maybe as well,
            // so that we trigger decomposition and recomposition.
            norm.combinesBack=true;
            didDecompose|=true;
        }
        if(cNorm.hasMapping()) {
            if(norm.mappingType==Norm::ROUND_TRIP) {
                if(prev==0) {
                    if(cNorm.mappingType!=Norm::ROUND_TRIP) {
                        fprintf(stderr,
                                "gennorm2 error: "
                                "U+%04lX's round-trip mapping's starter "
                                "U+%04lX one-way-decomposes, "
                                "not possible in Unicode normalization\n",
                                static_cast<long>(start), static_cast<long>(c));
                        exit(U_INVALID_FORMAT_ERROR);
                    }
                    uint8_t myTrailCC=norms.getCC(m.char32At(i));
                    UChar32 cTrailChar=cNorm.mapping->char32At(cNorm.mapping->length()-1);
                    uint8_t cTrailCC=norms.getCC(cTrailChar);
                    if(cTrailCC>myTrailCC) {
                        fprintf(stderr,
                                "gennorm2 error: "
                                "U+%04lX's round-trip mapping's starter "
                                "U+%04lX decomposes and the "
                                "inner/earlier tccc=%hu > outer/following tccc=%hu, "
                                "not possible in Unicode normalization\n",
                                static_cast<long>(start), static_cast<long>(c),
                                static_cast<short>(cTrailCC), static_cast<short>(myTrailCC));
                        exit(U_INVALID_FORMAT_ERROR);
                    }
                } else {
                    fprintf(stderr,
                            "gennorm2 error: "
                            "U+%04lX's round-trip mapping's non-starter "
                            "U+%04lX decomposes, "
                            "not possible in Unicode normalization\n",
                            static_cast<long>(start), static_cast<long>(c));
                    exit(U_INVALID_FORMAT_ERROR);
                }
            }
            if(decomposed==nullptr) {
                decomposed=new UnicodeString(m, 0, prev);
            }
            decomposed->append(*cNorm.mapping);
        } else if(Hangul::isHangul(c)) {
            char16_t buffer[3];
            int32_t hangulLength=Hangul::decompose(c, buffer);
            if(norm.mappingType==Norm::ROUND_TRIP && prev!=0) {
                fprintf(stderr,
                        "gennorm2 error: "
                        "U+%04lX's round-trip mapping's non-starter "
                        "U+%04lX decomposes, "
                        "not possible in Unicode normalization\n",
                        static_cast<long>(start), static_cast<long>(c));
                exit(U_INVALID_FORMAT_ERROR);
            }
            if(decomposed==nullptr) {
                decomposed=new UnicodeString(m, 0, prev);
            }
            decomposed->append(buffer, hangulLength);
        } else if(decomposed!=nullptr) {
            decomposed->append(m, prev, i-prev);
        }
    }
    if(decomposed!=nullptr) {
        if(norm.rawMapping==nullptr) {
            // Remember the original mapping when decomposing recursively.
            norm.rawMapping=norm.mapping;
        } else {
            delete norm.mapping;
        }
        norm.mapping=decomposed;
        // Not  norm.setMappingCP();  because the original mapping
        // is most likely to be encodable as a delta.
        didDecompose|=true;
    }
}

U_NAMESPACE_END

#endif // #if !UCONFIG_NO_NORMALIZATION