/*
 * Copyright (c) 2007-2009, Dennis M. Sosnoski. All rights reserved.
 * 
 * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the
 * following conditions are met:
 * 
 * Redistributions of source code must retain the above copyright notice, this list of conditions and the following
 * disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the
 * following disclaimer in the documentation and/or other materials provided with the distribution. Neither the name of
 * JiBX nor the names of its contributors may be used to endorse or promote products derived from this software without
 * specific prior written permission.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package org.jibx.schema.codegen;

import java.io.BufferedWriter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.eclipse.jdt.core.dom.AST;
import org.jibx.binding.model.BindingHolder;
import org.jibx.binding.model.BindingOrganizer;
import org.jibx.binding.model.ElementBase;
import org.jibx.binding.model.FormatElement;
import org.jibx.binding.model.MappingElement;
import org.jibx.binding.model.MappingElementBase;
import org.jibx.binding.model.StructureElement;
import org.jibx.extras.DocumentComparator;
import org.jibx.runtime.BindingDirectory;
import org.jibx.runtime.IBindingFactory;
import org.jibx.runtime.IMarshallingContext;
import org.jibx.runtime.IUnmarshallable;
import org.jibx.runtime.IUnmarshallingContext;
import org.jibx.runtime.JiBXException;
import org.jibx.runtime.QName;
import org.jibx.runtime.Utility;
import org.jibx.runtime.impl.UnmarshallingContext;
import org.jibx.schema.INamed;
import org.jibx.schema.SchemaUtils;
import org.jibx.schema.SchemaVisitor;
import org.jibx.schema.TreeWalker;
import org.jibx.schema.UrlResolver;
import org.jibx.schema.codegen.custom.ComponentExtension;
import org.jibx.schema.codegen.custom.GlobalExtension;
import org.jibx.schema.codegen.custom.SchemaCustom;
import org.jibx.schema.codegen.custom.SchemaExtension;
import org.jibx.schema.codegen.custom.SchemasetCustom;
import org.jibx.schema.codegen.extend.ClassDecorator;
import org.jibx.schema.codegen.extend.NameConverter;
import org.jibx.schema.elements.AnnotatedBase;
import org.jibx.schema.elements.CommonTypeDerivation;
import org.jibx.schema.elements.ElementElement;
import org.jibx.schema.elements.OpenAttrBase;
import org.jibx.schema.elements.SchemaBase;
import org.jibx.schema.elements.SchemaElement;
import org.jibx.schema.elements.SimpleRestrictionElement;
import org.jibx.schema.elements.UnionElement;
import org.jibx.schema.support.SchemaTypes;
import org.jibx.schema.validation.PrevalidationVisitor;
import org.jibx.schema.validation.RegistrationVisitor;
import org.jibx.schema.validation.ValidationContext;
import org.jibx.schema.validation.ValidationProblem;
import org.jibx.schema.validation.ValidationVisitor;
import org.jibx.util.InsertionOrderedSet;
import org.jibx.util.LazyList;
import org.xmlpull.v1.XmlPullParserException;

/**
 * Code generator from schema definition.
 * 
 * @author Dennis M. Sosnoski
 */
public class CodeGen
{
    /** Logger for class. */
    private static final Logger s_logger = Logger.getLogger(CodeGen.class.getName());
    
    /** Default type replacements applied. */
    private static final QName[] DEFAULT_REPLACEMENTS =
        new QName[] {
            SchemaTypes.ANY_URI.getQName(), SchemaTypes.STRING.getQName(),
            SchemaTypes.DURATION.getQName(), SchemaTypes.STRING.getQName(),
            SchemaTypes.ENTITIES.getQName(), SchemaTypes.STRING.getQName(),
            SchemaTypes.ENTITY.getQName(), SchemaTypes.STRING.getQName(),
            SchemaTypes.GDAY.getQName(), SchemaTypes.STRING.getQName(),
            SchemaTypes.GMONTH.getQName(), SchemaTypes.STRING.getQName(),
            SchemaTypes.GMONTHDAY.getQName(), SchemaTypes.STRING.getQName(),
            SchemaTypes.GYEAR.getQName(), SchemaTypes.STRING.getQName(),
            SchemaTypes.GYEARMONTH.getQName(), SchemaTypes.STRING.getQName(),
            SchemaTypes.ID.getQName(), SchemaTypes.STRING.getQName(),
            SchemaTypes.IDREF.getQName(), SchemaTypes.STRING.getQName(),
            SchemaTypes.IDREFS.getQName(), SchemaTypes.STRING.getQName(),
            SchemaTypes.LANGUAGE.getQName(), SchemaTypes.STRING.getQName(),
            SchemaTypes.NAME.getQName(), SchemaTypes.STRING.getQName(),
            SchemaTypes.NEGATIVE_INTEGER.getQName(), SchemaTypes.INTEGER.getQName(),
            SchemaTypes.NON_NEGATIVE_INTEGER.getQName(), SchemaTypes.INTEGER.getQName(),
            SchemaTypes.NON_POSITIVE_INTEGER.getQName(), SchemaTypes.INTEGER.getQName(),
            SchemaTypes.NORMALIZED_STRING.getQName(), SchemaTypes.STRING.getQName(),
            SchemaTypes.NCNAME.getQName(), SchemaTypes.STRING.getQName(),
            SchemaTypes.NMTOKEN.getQName(), SchemaTypes.STRING.getQName(),
            SchemaTypes.NMTOKENS.getQName(), SchemaTypes.STRING.getQName(),
            SchemaTypes.NOTATION.getQName(), SchemaTypes.STRING.getQName(),
            SchemaTypes.POSITIVE_INTEGER.getQName(), SchemaTypes.STRING.getQName(),
            SchemaTypes.TOKEN.getQName(), SchemaTypes.STRING.getQName(),
            SchemaTypes.UNSIGNED_BYTE.getQName(), SchemaTypes.BYTE.getQName(),
            SchemaTypes.UNSIGNED_INT.getQName(), SchemaTypes.INT.getQName(),
            SchemaTypes.UNSIGNED_LONG.getQName(), SchemaTypes.LONG.getQName(),
            SchemaTypes.UNSIGNED_SHORT.getQName(), SchemaTypes.SHORT.getQName()
        };
    
    /** Mask for schema elements which derive from a type. */
    private static final long TYPE_DERIVE_MASK =
        SchemaBase.ELEMENT_MASKS[SchemaBase.EXTENSION_TYPE] | SchemaBase.ELEMENT_MASKS[SchemaBase.RESTRICTION_TYPE];
    
    /** Mask for schema elements which define a type. */
    private static final long TYPE_DEFINE_MASK =
        SchemaBase.ELEMENT_MASKS[SchemaBase.COMPLEXTYPE_TYPE] | SchemaBase.ELEMENT_MASKS[SchemaBase.SIMPLETYPE_TYPE];
    
    /** Mask for schema elements which block name inheritance downward. */
    private static final long BLOCK_NAME_INHERIT_MASK =
        TYPE_DERIVE_MASK | SchemaBase.ELEMENT_MASKS[SchemaBase.UNION_TYPE];
    
    /** Code generation customizations. */
    private final SchemasetCustom m_global;
    
    /** Root URL for schemas. */
    private final URL m_schemaRoot;
    
    /** Root directory for schemas. */
    private final File m_schemaDir;
    
    /** Target directory for code generation. */
    private final File m_targetDir;
    
    /** Context for loading and processing schemas. */
    private final ValidationContext m_validationContext;
    
    /** Package directory for generated classes. */
    private PackageOrganizer m_packageDirectory;
    
    /** Definitions to be generated (may be global schema definitions, or reused nested components with classes). */
    private ArrayList m_definitions;
    
    /** Directory for constructed bindings. */
    private BindingOrganizer m_bindingDirectory;
    
    /**
     * Constructor.
     * 
     * @param parms command line parameters
     */
    public CodeGen(CodeGenCommandLine parms) {
        m_global = parms.getCustomRoot();
        addDefaultSubstitutions(m_global);
        m_schemaRoot = parms.getSchemaRoot();
        m_schemaDir = parms.getSchemaDir();
        m_targetDir = parms.getGeneratePath();
        m_validationContext = new ValidationContext();
    }
    
    /**
     * Constructor used by tests. This uses supplied schemas and skips writing to the file system.
     * 
     * @param custom
     * @param vctx
     */
    public CodeGen(SchemasetCustom custom, ValidationContext vctx) {
        m_global = custom;
        addDefaultSubstitutions(custom);
        m_schemaRoot = null;
        m_schemaDir = null;
        m_targetDir = null;
        m_validationContext = vctx;
    }
    
    /**
     * Add default type substitutions to set currently defined.
     *
     * @param custom
     */
    private static void addDefaultSubstitutions(SchemasetCustom custom) {
        QName[] subs = custom.getSubstitutions();
        if (subs == null) {
            custom.setSubstitutions(DEFAULT_REPLACEMENTS);
        } else {
            QName[] newsubs = new QName[subs.length + DEFAULT_REPLACEMENTS.length];
            System.arraycopy(subs, 0, newsubs, 0, subs.length);
            System.arraycopy(DEFAULT_REPLACEMENTS, 0, newsubs, subs.length, DEFAULT_REPLACEMENTS.length);
            custom.setSubstitutions(newsubs);
        }
    }
    
    /**
     * Find the most specific schemaset owning a schema. If multiple matches are found which are not in line of
     * containment the first match is returned and the conflict is reported as an error.
     * 
     * @param schema
     * @param custom schema set customization
     * @return owning schemaset, <code>null</code> if none
     */
    private SchemasetCustom findSchemaset(SchemaElement schema, SchemasetCustom custom) {
        LazyList childs = custom.getChildren();
        SchemasetCustom owner = null;
        String name = schema.getResolver().getName();
        for (int i = 0; i < childs.size(); i++) {
            Object child = childs.get(i);
            if (child instanceof SchemasetCustom) {
                SchemasetCustom schemaset = (SchemasetCustom)child;
                if (schemaset.isInSet(name, schema)) {
                    SchemasetCustom match = findSchemaset(schema, schemaset);
                    if (match != null) {
                        if (owner == null) {
                            owner = match;
                        } else {
                            m_validationContext.addError("schema-set overlap on schema " + name + " (first match "
                                + ValidationProblem.componentDescription(owner) + ')', match);
                        }
                    }
                }
            }
        }
        return owner == null ? custom : owner;
    }
    
    /**
     * Validate the schemas.
     * 
     * @param schemas schemas to be validated
     */
    public void validateSchemas(SchemaElement[] schemas) {
        
        // start by clearing name registers for previously-validated schemas
        for (Iterator iter = m_validationContext.iterateSchemas(); iter.hasNext();) {
            ((SchemaElement)iter.next()).getRegister().reset();
        }
        
        // run full schema validation
        TreeWalker wlkr = new TreeWalker(m_validationContext, m_validationContext);
        s_logger.debug("Beginning schema prevalidation pass");
        m_validationContext.clearTraversed();
        s_logger.debug("Beginning schema prevalidation pass");
        for (int i = 0; i < schemas.length; i++) {
            wlkr.walkSchema(schemas[i], new PrevalidationVisitor(m_validationContext));
            s_logger.debug("After prevalidation schema " + schemas[i].getResolver().getName() +
                " has effective namespace " + schemas[i].getEffectiveNamespace());
        }
        s_logger.debug("Beginning schema registration pass");
        m_validationContext.clearTraversed();
        for (int i = 0; i < schemas.length; i++) {
            wlkr.walkSchema(schemas[i], new RegistrationVisitor(m_validationContext));
        }
        s_logger.debug("Beginning validation pass");
        m_validationContext.clearTraversed();
        for (int i = 0; i < schemas.length; i++) {
            wlkr.walkSchema(schemas[i], new ValidationVisitor(m_validationContext));
            s_logger.debug("After validation schema " + schemas[i].getResolver().getName() +
                " has effective namespace " + schemas[i].getEffectiveNamespace());
        }
    }
    
    /**
     * Load and validate the root schema list.
     * 
     * @param list resolvers for schemas to be loaded
     * @return schemas in validation order
     * @throws JiBXException on unrecoverable error in schemas
     * @throws IOException on error reading schemas
     */
    private SchemaElement[] load(ArrayList list) throws JiBXException, IOException {
        IBindingFactory factory = BindingDirectory.getFactory(SchemaUtils.XS_PREFIX_BINDING, SchemaElement.class);
        IUnmarshallingContext ictx = factory.createUnmarshallingContext();
        int count = list.size();
        SchemaElement[] schemas = new SchemaElement[count];
        for (int i = 0; i < count; i++) {
            
            // unmarshal document to construct schema structure
            UrlResolver resolver = (UrlResolver)list.get(i);
            ictx.setDocument(resolver.getContent(), resolver.getName(), null);
            ictx.setUserContext(m_validationContext);
            SchemaElement schema = new SchemaElement();
            ((IUnmarshallable)schema).unmarshal(ictx);
            
            // set resolver for use during schema processing
            schemas[i] = schema;
            schema.setResolver(resolver);
            String id = resolver.getId();
            m_validationContext.setSchema(id, schema);
            
            // verify schema roundtripping if debug enabled
            if (s_logger.isDebugEnabled()) {
                try {
                    
                    // determine encoding of input document
                    String enc = ((UnmarshallingContext)ictx).getInputEncoding();
                    if (enc == null) {
                        enc = "UTF-8";
                    }
                    
                    // marshal root object back out to document in memory
                    IMarshallingContext mctx = factory.createMarshallingContext();
                    ByteArrayOutputStream bos = new ByteArrayOutputStream();
                    mctx.setIndent(2);
                    mctx.marshalDocument(schema, "UTF-8", null, bos);
                    
                    // compare with original input document
                    InputStreamReader brdr =
                        new InputStreamReader(new ByteArrayInputStream(bos.toByteArray()), "UTF-8");
                    InputStreamReader frdr = new InputStreamReader(resolver.getContent(), enc);
                    ByteArrayOutputStream baos = new ByteArrayOutputStream();
                    PrintStream pstream = new PrintStream(baos);
                    DocumentComparator comp = new DocumentComparator(pstream);
                    if (comp.compare(frdr, brdr)) {
                        
                        // report schema roundtripped successfully
                        s_logger.debug("Successfully roundtripped schema " + id);
                        
                    } else {
                        
                        // report problems in roundtripping schema
                        s_logger.debug("Errors in roundtripping schema " + id);
                        pstream.flush();
                        s_logger.debug(baos.toString());
                        
                    }
                    
                } catch (XmlPullParserException e) {
                    s_logger.debug("Error during schema roundtripping", e);
                }
            }
        }
        
        // to correctly handle namespaces for includes, process namespaced schemas first
        ArrayList ordereds = new ArrayList();
        m_validationContext.clearTraversed();
        for (int i = 0; i < count; i++) {
            SchemaElement schema = schemas[i];
            if (schema.getTargetNamespace() != null) {
                ordereds.add(schema);
            }
        }
        for (int i = 0; i < count; i++) {
            SchemaElement schema = schemas[i];
            if (schema.getTargetNamespace() == null) {
                ordereds.add(schema);
            }
        }
        
        // add element definitions to schema if requested
//        if (ClassHolder.ADD_GLOBAL_ELEMENTS) {
//            for (int i = 0; i < ordereds.size(); i++) {
//                SchemaElement schema = (SchemaElement)ordereds.get(i);
//                ArrayList elementdefs = new ArrayList();
//                FilteredSegmentList childs = schema.getTopLevelChildren();
//                for (int j = 0; j < childs.size(); j++) {
//                    SchemaBase child = (SchemaBase)childs.get(j);
//                    if (child.type() == SchemaBase.COMPLEXTYPE_TYPE) {
//                        ElementElement elementdef = new ElementElement();
//                        String name = ((ComplexTypeElement)child).getName();
//                        elementdef.setType(new QName(schema.getEffectiveNamespace(), name));
//                        elementdef.setName(name);
//                        elementdefs.add(elementdef);
//                    }
//                }
//                if (elementdefs.size() > 0) {
//                    childs.addAll(elementdefs);
//                    s_logger.debug("Added " + elementdefs.size() + " element definitions for complexTypes to schema " + schema.getResolver().getName());
//                }
//            }
//        }
        
        // validate the schemas in order
        SchemaElement[] ordschemas = (SchemaElement[])ordereds.toArray(new SchemaElement[ordereds.size()]);
        validateSchemas(ordschemas);
        return ordschemas;
    }
    
    /**
     * Validate and apply customizations to loaded schemas.
     * 
     * @return <code>true</code> if successful, <code>false</code> if error
     */
    public boolean customizeSchemas() {
        
        // TODO: remove this once union handling fully implemented
        SchemaVisitor visitor = new SchemaVisitor() {
            
            public void exit(UnionElement node) {
                OpenAttrBase parent = node.getParent();
                int count = parent.getChildCount();
                for (int i = 0; i < count; i++) {
                    if (parent.getChild(i) == node) {
                        SimpleRestrictionElement empty = new SimpleRestrictionElement();
                        empty.setBase(SchemaTypes.STRING.getQName());
                        parent.replaceChild(i, empty);
                        break;
                    }
                }
            }
            
        };
        TreeWalker wlkr = new TreeWalker(null, null);
        for (Iterator iter = m_validationContext.iterateSchemas(); iter.hasNext();) {
            wlkr.walkElement((SchemaElement)iter.next(), visitor);
        }
        
//        // TODO: test code for annotating schema
//        try {
//            DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
//            final Document document = builder.newDocument();
//            visitor = new SchemaVisitor() {
//                
//                public void exit(AnnotatedBase node) {
//                    if (node.getAnnotation() == null) {
//                        StringBuffer buff = new StringBuffer();
//                        buff.append(node.name());
//                        if (node instanceof INamed && ((INamed)node).getName() != null) {
//                            buff.append(" name='");
//                            buff.append(((INamed)node).getName());
//                            buff.append('\'');
//                        } else if (node instanceof IReference && ((IReference)node).getRef() != null) {
//                            buff.append(" ref='");
//                            buff.append(((IReference)node).getRef().getName());
//                            buff.append('\'');
//                        }
//                        AnnotationElement anno = new AnnotationElement();
//                        node.setAnnotation(anno);
//                        DocumentationElement doc = new DocumentationElement();
//                        doc.addContent(document.createTextNode(buff.toString()));
//                        anno.getItemsList().add(doc);
//                    }
//                }
//                
//            };
//            for (Iterator iter = m_validationContext.iterateSchemas(); iter.hasNext();) {
//                wlkr.walkElement((SchemaElement)iter.next(), visitor);
//            }
//        } catch (ParserConfigurationException e) {
//            /* ignore any error */
//        }
        
        // validate the customizations
        m_global.validate(m_validationContext);
        
        // create the package directory, using default package from root schemaset
        String dfltpack = m_global.getPackage();
        if (dfltpack == null) {
            dfltpack = "";
        }
        m_packageDirectory = new PackageOrganizer(m_targetDir, dfltpack);
        
        // link each schema to a customization, creating a default customization if necessary
        int count = 0;
        for (Iterator iter = m_validationContext.iterateSchemas(); iter.hasNext();) {
            SchemaElement schema = (SchemaElement)iter.next();
            s_logger.debug("Assigning customization for schema " + ++count + ": " + schema.getResolver().getName());
            SchemasetCustom owner = findSchemaset(schema, m_global);
            SchemaCustom custom = owner.forceCustomization(schema.getResolver().getName(), schema, m_validationContext);
            custom.validate(m_validationContext);
            String pname = custom.getPackage();
            PackageHolder holder = null;
            if (pname == null) {
                String uri = schema.getEffectiveNamespace();
                if (uri == null) {
                    uri = "";
                }
                holder = m_packageDirectory.getPackageForUri(uri);
            } else {
                holder = m_packageDirectory.getPackage(pname);
            }
            custom.extend(custom.getNameConverter(), custom.getClassDecorators(), holder, m_validationContext);
        }
        
        // check all the customizations
        m_global.checkSchemas(m_validationContext);
        return !reportProblems(m_validationContext);
    }
    
    /**
     * Process substitutions and deletions defined by extensions. This builds the cross-reference information for the
     * global definition components of the schemas while removing references to deleted components.
     * 
     * @return <code>true</code> if any changes to the schemas, <code>false</code> if not
     */
    private boolean processExtensions() {
        
        // first clear all the cross reference information
        for (Iterator iter = m_validationContext.iterateSchemas(); iter.hasNext();) {
            SchemaElement schema = (SchemaElement)iter.next();
            int count = schema.getChildCount();
            for (int i = 0; i < count; i++) {
                SchemaBase child = schema.getChild(i);
                Object obj = child.getExtension();
                if (obj instanceof GlobalExtension) {
                    ((GlobalExtension)obj).resetDependencies();
                }
            }
        }
        
        // process each loaded schema for deletions and cross referencing
        int index = 0;
        m_validationContext.clearTraversed();
        boolean modified = false;
        // Level level = TreeWalker.setLogging(s_logger.getLevel());
        for (Iterator iter = m_validationContext.iterateSchemas(); iter.hasNext();) {
            SchemaElement schema = (SchemaElement)iter.next();
            m_validationContext.enterSchema(schema);
            s_logger.debug("Applying extensions to schema " + ++index + ": " + schema.getResolver().getName());
            int count = schema.getChildCount();
            boolean instmod = false;
            for (int i = 0; i < count; i++) {
                SchemaBase child = schema.getChild(i);
                Object obj = child.getExtension();
                if (obj instanceof GlobalExtension) {
                    
                    // apply extension to global definition element
                    ComponentExtension exten = (ComponentExtension)obj;
                    if (exten.isRemoved()) {
                        
                        // just eliminate this definition from the schema
                        schema.detachChild(i);
                        instmod = true;
                        
                    } else {
                        
                        // process the definition to remove references to deleted components
                        exten.applyAndCountUsage(m_validationContext);
                        
                    }
                }
            }
            if (instmod) {
                schema.compactChildren();
                modified = true;
            }
            m_validationContext.exitSchema();
            
        }
        // TreeWalker.setLogging(level);
        return modified;
    }

    /**
     * Apply extensions and normalize all schemas. This may be a multipass process, since applying extensions may create
     * the opportunity for further normalizations and vice versa.
     */
    public void applyAndNormalize() {
        
        // loop until no modifications, with at least one pass of extensions and normalizations
        boolean modified = true;
        while (processExtensions() || modified) {
            
            // normalize all the schema definitions
            modified = false;
            for (Iterator iter = m_validationContext.iterateSchemas(); iter.hasNext();) {
                SchemaElement schema = (SchemaElement)iter.next();
                int count = schema.getChildCount();
                boolean instmod = false;
                for (int i = 0; i < count; i++) {
                    SchemaBase child = schema.getChild(i);
                    Object obj = child.getExtension();
                    if (obj instanceof GlobalExtension) {
                        GlobalExtension global = (GlobalExtension)obj;
                        global.normalize();
                        if (global.isRemoved() ||
                            (!global.isIncluded() && global.isPreferInline() && global.getOverrideType() != null)) {
                            
                            // just eliminate this definition from the schema
                            schema.detachChild(i);
                            instmod = true;
                            
                        }
                    }
                }
                if (instmod) {
                    schema.compactChildren();
                    modified = true;
                }
            }
            
        }
        
        // finish by flagging global definitions requiring separate classes
        for (Iterator iter = m_validationContext.iterateSchemas(); iter.hasNext();) {
            SchemaElement schema = (SchemaElement)iter.next();
            SchemaCustom custom = ((SchemaExtension)schema.getExtension()).getCustom();
            if (custom.isGenerateAll()) {
                int count = schema.getChildCount();
                for (int i = 0; i < count; i++) {
                    SchemaBase comp = schema.getChild(i);
                    if (comp.type() == SchemaBase.ELEMENT_TYPE || comp.type() == SchemaBase.COMPLEXTYPE_TYPE) {
                        Object obj = comp.getExtension();
                        if (obj instanceof GlobalExtension) {
                            ((GlobalExtension)obj).setIncluded(true);
                            if (s_logger.isDebugEnabled()) {
                                s_logger.debug("Set include for definition " + SchemaUtils.describeComponent(comp));
                            }
                        }
                    }
                }
            }
        }
    }

    /**
     * Processes the schemas to remove unused global definitions.
     */
    public void pruneDefinitions() {
        
        // start by recursively checking for removable global definitions
        int index = 0;
        for (Iterator iter = m_validationContext.iterateSchemas(); iter.hasNext();) {
            SchemaElement schema = (SchemaElement)iter.next();
            s_logger.debug("Checking for unused definitions in schema " + ++index + ": "
                + schema.getResolver().getName());
            int count = schema.getChildCount();
            for (int i = 0; i < count; i++) {
                SchemaBase child = schema.getChild(i);
                Object exten = child.getExtension();
                if (exten instanceof GlobalExtension) {
                    
                    // check if global definition is unused and not specifically required
                    ((GlobalExtension)exten).checkRemovable();
                    
                }
            }
        }
        
        // next remove all the definitions flagged in the first step
        index = 0;
        for (Iterator iter = m_validationContext.iterateSchemas(); iter.hasNext();) {
            SchemaElement schema = (SchemaElement)iter.next();
            s_logger.debug("Deleting unused definitions in schema " + ++index + ": " + schema.getResolver().getName());
            int count = schema.getChildCount();
            boolean modified = false;
            for (int i = 0; i < count; i++) {
                SchemaBase child = schema.getChild(i);
                Object exten = child.getExtension();
                if (exten instanceof GlobalExtension && ((ComponentExtension)exten).isRemoved()) {
                    
                    // remove the definition from schema
                    schema.detachChild(i);
                    modified = true;
                    if (s_logger.isDebugEnabled()) {
                        s_logger.debug(" Removed definition " + ((INamed)child).getQName());
                    }
                    
                }
            }
            if (modified) {
                schema.compactChildren();
            }
        }
    }
    
    /**
     * Check if an item has an associated name. If the component associated with the item has a name, this just returns
     * that name. The only exception is for inlined global type definitions, which are treated as unnamed.
     * 
     * @param item
     * @return name associated name, or <code>null</code> if none
     */
    private String checkDirectName(Item item) {
        AnnotatedBase comp = item.getSchemaComponent();
        if (comp instanceof INamed) {
            
            // check for an inlined global type definition
            boolean usename = true;
            if (comp.isGlobal()) {
                if ((comp.bit() & TYPE_DEFINE_MASK) != 0 && !(item instanceof DefinitionItem)) {
                    usename = false;
                }
            }
            if (usename) {
                
                // use name from schema component
                String name = ((INamed)comp).getName();
                if (name != null) {
                    NameConverter nconv = item.getComponentExtension().getGlobal().getNameConverter();
                    return nconv.toBaseName(nconv.trimXName(name));
                }
            }
        }
        return null;
    }
    
    /**
     * Derive the base name for an item. If not forced, the only time a name will be returned is when the item is a
     * reference to a non-type definition. If forced, this will try other alternatives for names including the text
     * "Enumeration" for an enumeration group, the base type name for a type derivation, the schema type name for a
     * value of a schema type, or finally the schema component element name.
     * 
     * @param item
     * @param force name forced flag
     * @return name (<code>null</code> if to use inherited name when <code>force == false</code>)
     */
    private String deriveName(Item item, boolean force) {
        
        // try alternatives, in decreasing preference order
        AnnotatedBase comp = item.getSchemaComponent();
        String text = null;
        if (force) {
            
            if (item instanceof ReferenceItem) {
                text = ((ReferenceItem)item).getDefinition().getName();
            } else if (item instanceof GroupItem && ((GroupItem)item).isEnumeration()) {
                text = "Enumeration";
            } else if ((TYPE_DERIVE_MASK & comp.bit()) != 0) {
                text = ((CommonTypeDerivation)comp).getBase().getName();
            } else if (item instanceof ValueItem) {
                text = ((ValueItem)item).getSchemaType().getName();
            } else {
                text = comp.name();
            }
            
        } else if (item instanceof ReferenceItem && (TYPE_DEFINE_MASK & comp.type()) == 0) {
            
            // use name from definition in the case of anything except a type definition
            text = ((ReferenceItem)item).getDefinition().getName();
        }
        
        if (text == null) {
            return null;
        } else {
            return item.getComponentExtension().getGlobal().getNameConverter().toBaseName(text);
        }
    }
    
    /**
     * Compact group structures. This eliminates redundant groupings, in the form of groups with only one child, which
     * child is a group referencing the same schema component as the parent group, from the data structure
     * representation.
     *
     * @param group
     */
    private void compactGroups(GroupItem group) {
        Item child;
        while (group.getChildCount() == 1 && (child = group.getFirstChild()) instanceof GroupItem &&
            !child.isTopmost()) {
            group.adoptChildren((GroupItem)child);
        }
        for (child = group.getFirstChild(); child != null; child = child.getNext()) {
            if (child instanceof GroupItem) {
                compactGroups((GroupItem)child);
            }
        }
    }
    
    /**
     * Assemble a name from a base name and an optional prefix. If the prefix is supplied, this first converts the base
     * name to embedded form, then appends the prefix.
     *
     * @param prefix
     * @param base
     * @return name
     */
    private String assembleName(String prefix, String base) {
        if (prefix == null) {
            return base;
        } else {
            return prefix + NameUtils.toNameWord(base);
        }
    }
    
    /**
     * Set the basic names to be used for a structure of items. For named components of the schema definition the names
     * used are simply the converted XML local names, for other components more complex rules apply (see {@link
     * #deriveName(Item,boolean)}. This method calls itself recursively to handle nested groups.
     * 
     * @param prefix text to be prefixed to names within inlined group (<code>null</code> if none)
     * @param group item group to be assigned
     * @param force force name derivation flag
     */
    private void assignNames(String prefix, GroupItem group, boolean force) {
        
        // use existing name if set, otherwise derive from context if necessary
        String name = group.getName();
        boolean propagate = group.getChildCount() == 1;
        if (name == null) {
            name = checkDirectName(group);
            if (name == null && force) {
                propagate = false;
                name = deriveName(group, true);
            }
        }
        
        // set name needed for this structure (as either value or class)
        if (name != null) {
            if (group.getClassName() == null) {
                group.setClassName(NameUtils.toNameWord(name));
            }
            if (group.getName() == null) {
                group.setName(NameUtils.toNameLead(assembleName(prefix, name)));
            }
        }
        
        // set passed-down prefix to this name if inline group with element name and more than one child, or multiple
        //  nested children and no passed-in prefix
        if (group.isInline() && name != null) {
            AnnotatedBase comp = group.getSchemaComponent();
            if (comp.type() == SchemaBase.ELEMENT_TYPE && SchemaUtils.isNamed(comp)) {
                boolean passprefix = false;
                if (group.getChildCount() > 1) {
                    passprefix = true;
                } else if (prefix == null) {
                    passprefix = true;
                    GroupItem childgroup = group;
                    while (childgroup.getChildCount() == 1) {
                        Item childchild = childgroup.getFirstChild();
                        if (childchild instanceof GroupItem) {
                            childgroup = (GroupItem)childchild;
                        } else {
                            passprefix = false;
                            break;
                        }
                    }
                }
                if (passprefix) {
                    prefix = assembleName(prefix, name);
                }
            }
        }
        
        // propagate name downward if group is inline and single nested item without its own name
        Item head = group.getFirstChild();
        if (propagate) {
            
            // name can be inherited, but continue recursion for child group
            if (head instanceof GroupItem) {
                GroupItem childgroup = (GroupItem)head;
                assignNames(childgroup.isInline() ? prefix : null, childgroup, false);
            }
            
        } else {
            
            // process all child items with definite name assignments
            for (Item item = head; item != null; item = item.getNext()) {
                if (item instanceof GroupItem) {
                    GroupItem childgroup = (GroupItem)item;
                    assignNames(childgroup.isInline() ? prefix : null, childgroup, true);
                } else {
                    if (item.getName() == null) {
                        String childname = checkDirectName(item);
                        if (childname == null) {
                            childname = deriveName(item, true);
                        }
                        item.setName(NameUtils.toNameLead(assembleName(prefix, childname)));
                    }
                }
            }
            
        }
    }
    
    /**
     * Compute the complexity of a structure. In order to find the complexity of a structure all items of the structure
     * must first be checked for inlining, which in turn requires checking their complexity. That makes this method
     * mutually recursive with {@link #checkInline(DefinitionItem, int)}.
     * 
     * @param group
     * @param depth nesting depth
     * @return complexity (0, 1, or 2 for anything more than a single value)
     */
    private int computeComplexity(GroupItem group, int depth) {
        if (s_logger.isDebugEnabled()) {
            s_logger.debug(SchemaUtils.getIndentation(depth) + "counting values for "
                + (group instanceof DefinitionItem ? "definition " : "group ")
                + SchemaUtils.describeComponent(group.getSchemaComponent()));
        }
        
        // count the actual values in the structure
        int count = 0;
        for (Item item = group.getFirstChild(); item != null; item = item.getNext()) {
            
            // get the schema customization information
            SchemaCustom custom = ((SchemaExtension)item.getSchemaComponent().getSchema().getExtension()).getCustom();
            
            // handle inlining of references
            if (item instanceof ReferenceItem) {
                
                // first make sure the definition has been checked for inlining
                ReferenceItem reference = (ReferenceItem)item;
                DefinitionItem definition = reference.getDefinition();
                checkInline(definition, depth + 1);
                if (definition.isInline()) {
                    
                    // convert the reference to an inline copy of the definition
                    item = reference.inlineReference();
                    if (s_logger.isDebugEnabled()) {
                        s_logger.debug(SchemaUtils.getIndentation(depth) + "converted reference to "
                            + SchemaUtils.describeComponent(definition.getSchemaComponent()) + " to inline group");
                    }
                    
                }
            }
            
            // handle actual item count, and inlining of child group (may be new, from converted reference)
            if (item instanceof GroupItem) {
                
                // check count for nested group
                GroupItem grpitem = (GroupItem)item;
                int grpcount = computeComplexity(grpitem, depth + 1);
                
                // avoid inlining if an enumeration, or an extension reference; or the nested group is optional or a
                //  collection and has more than one item (or a single item which is itself optional or a collection,
                //  which will be counted as multiple items); or the main group is a choice, and the nested group is a
                //  compositor with more than one element, and the first element is optional (or the first element child
                //  of that element, if inlined)
                boolean inline = true;
                if (grpitem.isEnumeration()) {
                    inline = false;
                } else if (grpitem.isCollection() || grpitem.isOptional()) {
                    if (grpcount > 1) {
                        inline = false;
                    } else {
                        
                        // must be single child, but block inlining if that child is optional
                        Item child;
                        GroupItem childgrp = grpitem;
                        while ((child = childgrp.getFirstChild()) instanceof GroupItem) {
                            childgrp = (GroupItem)child;
                            if (childgrp.isOptional()) {
                                inline = false;
                                break;
                            }
                        }
                        
                    }
                } else if (grpcount > 1 && group.getSchemaComponent().type() == SchemaBase.CHOICE_TYPE) {
                    
                    // assume no inlining, but dig into structure to make sure first non-inlined element is required
                    inline = false;
                    Item child = grpitem.getFirstChild();
                    while (!child.isOptional()) {
                        if (child.getSchemaComponent().type() == SchemaBase.ELEMENT_TYPE &&
                            !(child instanceof GroupItem && ((GroupItem)child).isInline())) {
                            
                            // required element with simple value or separate class, safe to inline
                            inline = true;
                            break;
                            
                        } else if (child instanceof GroupItem) {
                            child = ((GroupItem)child).getFirstChild();
                            if (child == null) {
                                
                                // empty group, safe to inline
                                inline = true;
                                break;
                                
                            }
                        } else {
                            
                            // required reference item, safe to inline
                            inline = true;
                            break;
                            
                        }
                    }
                    
                }
                if (inline) {
                    
                    // inline the group
                    grpitem.setInline(true);
                    count += grpcount;
                    if (s_logger.isDebugEnabled()) {
                        s_logger.debug(SchemaUtils.getIndentation(depth) + "inlining "
                            + (grpitem instanceof DefinitionItem ? "definition " : "group ")
                            + SchemaUtils.describeComponent(grpitem.getSchemaComponent()) + " with item count " +
                            count);
                    }
                    
                } else {
                    
                    // force separate class for group
                    grpitem.setInline(false);
                    count++;
                    
                }
                
            } else {
                count++;
            }
            
            // bump up the complexity if the item is repeated (optionally inlining collection wrappers)
            if (!custom.isNullCollectionAllowed() && item.isCollection()) {
                count++;
            }
        }
        return count > 1 ? 2 : count;
    }
    
    /**
     * Check if a group consists only of a single non-repeating item, which is not an enumeration.
     * 
     * @param group
     * @return <code>true</code> if simple group, <code>false</code> if repeated, multiple, or enumeration
     */
    private boolean isSimple(GroupItem group) {
        if (group.isEnumeration() || group.getChildCount() > 1) {
            return false;
        } else {
            for (Item item = group.getFirstChild(); item != null; item = item.getNext()) {
                if (item.isCollection()) {
                    return false;
                } else if (item instanceof GroupItem && !isSimple((GroupItem)item)) {
                    return false;
                }
            }
            return true;
        }
    }
    
    /**
     * Convert nested groups which are not inlined to freestanding definitions. This calls itself recursively to process
     * nested groups, except those nested within groups converted to definitions.
     *
     * @param group
     */
    private void convertToDefinitions(GroupItem group) {
        Item item = group.getFirstChild();
        while (item != null) {
            if (item instanceof GroupItem) {
                GroupItem childgrp = (GroupItem)item;
                if (childgrp.isInline()) {
                    convertToDefinitions(childgrp);
                } else {
                    DefinitionItem definition = childgrp.convertToDefinition();
                    definition.setChecked(true);
                    OpenAttrBase ancestor = group.getSchemaComponent();
                    while (!ancestor.isGlobal()) {
                        ancestor = ancestor.getParent();
                    }
                    GlobalExtension global = (GlobalExtension)ancestor.getExtension();
                    PackageHolder pack = global.getPackage();
                    String clasname = definition.getClassName();
                    NameConverter nconv = global.getNameConverter();
                    boolean useinner = global.isUseInnerClasses();
                    ClassDecorator[] decorators = global.getClassDecorators();
                    ClassHolder clas = pack.addClass(clasname, nconv, decorators, useinner, childgrp.isEnumeration());
                    definition.setGenerateClass(clas);
                    m_definitions.add(definition);
                    s_logger.debug("Added definition class " + clas.getFullName());
                }
            }
            item = item.getNext();
        }
    }

    /**
     * Check if a global definition structure is to be inlined. This method is mutually recursive with {@link
     * #computeComplexity(GroupItem, int)}. The two methods together determine the inlining status of all items.
     * 
     * @param definition
     * @param depth nesting depth
     */
    private void checkInline(DefinitionItem definition, int depth) {
        if (!definition.isChecked()) {
            
            // initially say it won't be inlined, to avoid issues with circular references
            definition.setInline(false);
            definition.setChecked(true);
            if (s_logger.isDebugEnabled()) {
                s_logger.debug(SchemaUtils.getIndentation(depth) + "checking inlining of definition "
                    + SchemaUtils.describeComponent(definition.getSchemaComponent())
                    + (definition.isInlineBlocked() ? " (inlining blocked)" : ""));
            }
            
            // inline references where appropriate, and count the values defined
            int count = computeComplexity(definition, depth);
            if (!definition.isInlineBlocked() && ((count == 1 && isSimple(definition)) ||
                (((GlobalExtension)definition.getComponentExtension()).isPreferInline() &&
                definition.getReferenceCount() == 1))) {
                
                // set state as inlined
                definition.setInline(true);
                if (s_logger.isDebugEnabled()) {
                    s_logger.debug(SchemaUtils.getIndentation(depth) + "inlining definition "
                        + SchemaUtils.describeComponent(definition.getSchemaComponent()) +
                        " with item count " + count);
                }
                
                // convert non-inlined child components to definitions if multiple use
                if (definition.getReferenceCount() > 1) {
                    convertToDefinitions(definition);
                }
            }
        }
    }
    
    /**
     * Set the name and namespace URI for a concrete &lt;mapping> binding component. This is the same logic as used in
     * the {@link StructureClassHolder} equivalent.
     *
     * @param qname qualified name to be set (<code>null</code> if none)
     * @param mapping
     */
    private void setName(QName qname, MappingElementBase mapping) {
        if (qname != null) {
            String name = qname.getName();
            String uri = qname.getUri();
            BindingHolder holder = m_bindingDirectory.findBinding(uri);
            mapping.setName(name);
            if (!Utility.safeEquals(uri, holder.getElementDefaultNamespace())) {
                holder.addNamespaceDependency(uri);
                mapping.setUri(uri);
            }
        }
    }
    
    /**
     * Generate the data model. This first builds a representation of all the data items from the schema definitions,
     * then determines which items can be inlined and which need separate class representations.
     * 
     * @param verbose 
     * @return root package for binding
     */
    public String generate(boolean verbose) {
        
        // build the item structure for each definition
        ItemVisitor itembuilder = new ItemVisitor();
        int index = 0;
        ArrayList items = new ArrayList();
        ArrayList checkitems = new ArrayList();
        for (Iterator iter = m_validationContext.iterateSchemas(); iter.hasNext();) {
            SchemaElement schema = (SchemaElement)iter.next();
            m_validationContext.enterSchema(schema);
            s_logger.info("Building item structure for schema " + ++index + ": " + schema.getResolver().getName());
            int count = schema.getChildCount();
            for (int i = 0; i < count; i++) {
                SchemaBase child = schema.getChild(i);
                if (child.getExtension() instanceof GlobalExtension) {
                    
                    // create the definition
                    GlobalExtension global = (GlobalExtension)child.getExtension();
                    DefinitionItem definition = global.getDefinition();
                    if (definition == null) {
                        definition = itembuilder.buildGlobal((AnnotatedBase)child);
                        if (s_logger.isInfoEnabled()) {
                            s_logger.info("Constructed item structure for " + SchemaUtils.describeComponent(child)
                                + ":\n" + definition.describe());
                        }
                    } else if (s_logger.isInfoEnabled()) {
                        s_logger.info("Found existing item structure for " + SchemaUtils.describeComponent(child)
                            + ":\n" + definition.describe());
                    }
                    items.add(definition);
                    
                    // set the names on the definition so they'll be available for inlining
                    NameConverter nconv = global.getNameConverter();
                    String dfltname = nconv.toBaseName(nconv.trimXName(((INamed)child).getName()));
                    if (!definition.isFixedName()) {
                        String name = global.getBaseName();
                        if (name == null) {
                            name = NameUtils.toNameLead(dfltname);
                        }
                        definition.setName(name);
                    }
                    if (!definition.isFixedClassName()) {
                        String name = global.getClassName();
                        if (name == null) {
                            name = NameUtils.toNameWord(dfltname);
                        }
                        definition.setClassName(name);
                    }
                    
                    // force class generation if required
                    if (global.isIncluded()) {
                        definition.setInlineBlocked(true);
                        if (s_logger.isDebugEnabled()) {
                            s_logger.debug("Forcing class generation for " + SchemaUtils.describeComponent(child));
                        }
                    }
                    
                    // record all simple element definition items separately for next pass
                    if (child.type() == SchemaBase.ELEMENT_TYPE && definition.getChildCount() == 1) {
                        Item item = definition.getFirstChild();
                        if (item instanceof ReferenceItem) {
                            
                            // skip elements based on simpleTypes for this check, since simpleTypes never need a class
                            AnnotatedBase comp = ((ReferenceItem)item).getDefinition().getSchemaComponent();
                            if (comp.type() != SchemaBase.SIMPLETYPE_TYPE) {
                                checkitems.add(definition);
                            }
                        }
                    }
                }
            }
        }
        
        // check for special (but common) case of single global element using global complexType
        Map usemap = new HashMap();
        for (int i = 0; i < checkitems.size(); i++) {
            ReferenceItem reference = (ReferenceItem)((DefinitionItem)checkitems.get(i)).getFirstChild();
            DefinitionItem basedef = reference.getDefinition();
            Boolean usedirect = (Boolean)usemap.get(basedef);
            if (usedirect == null) {
                
                // first time type was referenced, flag to use directly
                usemap.put(basedef, Boolean.TRUE);
                
            } else if (usedirect.booleanValue()) {
                
                // second time type was referenced, flag to use separate classes
                usemap.put(basedef, Boolean.FALSE);
                
            }
        }
        HashMap typeinstmap = new HashMap();
        for (int i = 0; i < checkitems.size(); i++) {
            DefinitionItem definition = (DefinitionItem)checkitems.get(i);
            ReferenceItem reference = (ReferenceItem)definition.getFirstChild();
            DefinitionItem basedef = reference.getDefinition();
            if (((Boolean)usemap.get(basedef)).booleanValue()) {
                
                // single element definition using type, force class for type but none for element
                basedef.setInlineBlocked(true);
                definition.setInlineBlocked(false);
                definition.setInline(true);
                typeinstmap.put(basedef, definition);
                if (s_logger.isDebugEnabled()) {
                    s_logger.debug("Forcing inlining of type-isomorphic "
                        + SchemaUtils.describeComponent(definition.getSchemaComponent()));
                }
                
            }
        }
        
        // compact all items
        for (int i = 0; i < items.size(); i++) {
            DefinitionItem definition = (DefinitionItem)items.get(i);
            compactGroups(definition);
        }
        
        // inline references where appropriate
        m_definitions = new ArrayList();
        for (int i = 0; i < items.size(); i++) {
            
            // make sure definition has been checked for inlining (may add new definitions directly to list)
            DefinitionItem definition = (DefinitionItem)items.get(i);
            checkInline(definition, 1);
            
            // add definition to list if not inlined
            if (!definition.isInline()) {
                m_definitions.add(definition);
                GlobalExtension global = (GlobalExtension)definition.getComponentExtension();
                PackageHolder pack = global.getPackage();
                String cname = definition.getClassName();
                NameConverter nconv = global.getNameConverter();
                ClassDecorator[] decorators = global.getClassDecorators();
                boolean userinner = global.isUseInnerClasses();
                ClassHolder clas = pack.addClass(cname, nconv, decorators, userinner, definition.isEnumeration());
                definition.setGenerateClass(clas);
            }
        }
        
        // assign class and property names for all items
        for (int i = 0; i < m_definitions.size(); i++) {
            DefinitionItem definition = (DefinitionItem)m_definitions.get(i);
            compactGroups(definition);
            assignNames(null, definition, true);
            if (s_logger.isInfoEnabled()) {
                s_logger.info("After assigning names for "
                    + SchemaUtils.describeComponent(definition.getSchemaComponent()) + ":\n" + definition.describe());
            }
        }
        
        // convert extension references for all global type definitions
        for (int i = 0; i < m_definitions.size(); i++) {
            ((DefinitionItem)m_definitions.get(i)).convertTypeReference();
        }
        
        // classify all the items by form of content
        for (int i = 0; i < items.size(); i++) {
            DefinitionItem definition = (DefinitionItem)items.get(i);
            definition.classifyContent();
            if (s_logger.isInfoEnabled()) {
                s_logger.info("After inlining and classification for " +
                    SchemaUtils.describeComponent(definition.getSchemaComponent())
                    + ":\n" + definition.describe());
            }
        }
        
        // check namespace usage across all schemas
        SchemaNameVisitor visitor = new SchemaNameVisitor();
        TreeWalker walker = new TreeWalker(null, null);
        for (Iterator iter = m_validationContext.iterateSchemas(); iter.hasNext();) {
            walker.walkSchema((SchemaElement)iter.next(), visitor);
        }
        
        // build the actual class and binding structure
        boolean usenns = visitor.isNoNamespaceUsed();
        m_bindingDirectory = new BindingOrganizer(false, false, false, true, true, !usenns, !usenns);
        for (int i = 0; i < m_definitions.size(); i++) {
            
            // compact again after inlining and converting extension references
            DefinitionItem definition = (DefinitionItem)m_definitions.get(i);
            compactGroups(definition);
            
            // build the binding component for this definition
            ClassHolder clas = definition.getGenerateClass();
            OpenAttrBase comp = definition.getSchemaComponent();
            ElementBase binding;
            QName qname = null;
            String uri = null;
            if (definition.isEnumeration()) {
                FormatElement format = new FormatElement();
                format.setTypeName(clas.getBindingName());
                ((EnumerationClassHolder)clas).setBinding(format);
                binding = format;
            } else {
                MappingElementBase mapping = new MappingElement();
                mapping.setClassName(clas.getBindingName());
                if (comp.type() == SchemaBase.ELEMENT_TYPE) {
                    qname = ((ElementElement)comp).getQName();
                    uri = qname.getUri();
                } else {
                    mapping.setAbstract(true);
                    mapping.setTypeQName(definition.getQName());
                }
                ((StructureClassHolder)clas).setBinding(mapping);
                binding = mapping;
            }
            if (uri == null) {
                while (!(comp instanceof INamed) || ((INamed)comp).getName() == null) {
                    comp = comp.getParent();
                }
                uri = ((INamed)comp).getQName().getUri();
            }
            
            // add the mapping to appropriate binding
            BindingHolder holder = m_bindingDirectory.findBinding(uri);
            if (binding instanceof FormatElement) {
                m_bindingDirectory.addFormat((FormatElement)binding);
            } else {
                MappingElementBase mapping = (MappingElementBase)binding;
                holder.addMapping(mapping);
                if (!mapping.isAbstract()) {
                    setName(qname, mapping);
                }
                DefinitionItem elementdef = (DefinitionItem)typeinstmap.get(definition);
                if (elementdef != null) {
                    
                    // add concrete mapping for element name linked to type
                    ElementElement element = (ElementElement)elementdef.getSchemaComponent();
                    MappingElementBase concrete = new MappingElement();
                    concrete.setClassName(clas.getBindingName());
                    setName(element.getQName(), concrete);
                    StructureElement struct = new StructureElement();
                    qname = mapping.getTypeQName();
                    holder.addNamespaceDependency(qname.getUri());
                    struct.setMapAsQName(qname);
                    concrete.addChild(struct);
                    holder.addMapping(concrete);
                }
            }
            
            // set the definition for the class (and create any required secondary classes)
            clas.buildDataStructure(definition, holder);
        }
        
        // build the actual classes
        AST ast = AST.newAST(AST.JLS3);
        ArrayList packs = m_packageDirectory.getPackages();
        PackageHolder rootpack = null;
        for (int i = 0; i < packs.size(); i++) {
            PackageHolder pack = ((PackageHolder)packs.get(i));
            if (pack.getClassCount() > 0) {
                if (rootpack == null) {
                    PackageHolder scan = pack;
                    while (scan != null) {
                        if (scan.getClassCount() > 0 || scan.getSubpackageCount() > 1) {
                            rootpack = scan;
                        }
                        scan = scan.getParent();
                    }
                }
                pack.generate(verbose, ast, m_bindingDirectory);
            }
        }
        
        // write the binding definition(s)
        if (rootpack == null) {
            rootpack = m_packageDirectory.getPackage("");
        }
        return rootpack.getName();
    }
    
    /**
     * Get the binding directory.
     *
     * @return directory
     */
    public BindingOrganizer getBindingDirectory() {
        return m_bindingDirectory;
    }
    
    /**
     * Write the binding definitions file(s). This method can only be used after {@link #generate(boolean)} is called.
     *
     * @param name
     * @param pack
     * @throws JiBXException
     * @throws IOException
     */
    public void writeBindings(String name, String pack) throws JiBXException, IOException {
        m_bindingDirectory.configureFiles("binding.xml", new String[0], pack);
        m_bindingDirectory.writeBindings(m_targetDir);
    }

    /**
     * Report problems using console output. This clears the problem list after they've been reported, to avoid multiple
     * reports of the same problems.
     * 
     * @param vctx
     * @return <code>true</code> if one or more errors, <code>false</code> if not
     */
    private static boolean reportProblems(ValidationContext vctx) {
        ArrayList probs = vctx.getProblems();
        boolean error = false;
        if (probs.size() > 0) {
            for (int j = 0; j < probs.size(); j++) {
                ValidationProblem prob = (ValidationProblem)probs.get(j);
                String text;
                if (prob.getSeverity() >= ValidationProblem.ERROR_LEVEL) {
                    error = true;
                    text = "Error: " + prob.getDescription();
                    s_logger.error(text);
                } else {
                    text = "Warning: " + prob.getDescription();
                    s_logger.info(text);
                }
                System.out.println(text);
            }
        }
        probs.clear();
        return error;
    }
    
    /**
     * Add the schemas specified by customizations to the set to be loaded.
     * 
     * @param base root URL for schemas
     * @param dir root directory for schemas
     * @param custom schema set customization
     * @param fileset set of schema files to be loaded
     * @throws MalformedURLException
     */
/*    private static void addCustomizedSchemas(URL base, File dir, SchemasetCustom custom, InsertionOrderedSet fileset)
        throws MalformedURLException {
        
        // first check for name match patterns supplied
        String[] names = custom.getNames();
        if (names != null) {
            for (int i = 0; i < names.length; i++) {
                SchemaNameFilter filter = new SchemaNameFilter();
                String name = names[i];
                filter.setPattern(name);
                s_logger.debug("Matching file names to schemaset pattern '" + name + '\'');
                String[] matches = dir.list(filter);
                for (int j = 0; j < matches.length; j++) {
                    String match = matches[j];
                    fileset.add(new UrlResolver(match, new URL(base, match)));
                    s_logger.debug("Added schema from schemaset pattern match: " + match);
                }
            }
        }
        
        // next check all child customizations
        LazyList childs = custom.getChildren();
        for (int i = 0; i < childs.size(); i++) {
            Object child = childs.get(i);
            if (child instanceof SchemaCustom) {
                String name = ((SchemaCustom)child).getName();
                if (name != null) {
                    try {
                        fileset.add(new UrlResolver(name, new URL(base, name)));
                        s_logger.debug("Added schema from customizations: " + name);
                    } catch (MalformedURLException e) {
                        System.out.println("Error adding schema from customizations: " + name);
                    }
                }
            } else if (child instanceof SchemasetCustom) {
                addCustomizedSchemas(base, dir, (SchemasetCustom)child, fileset);
            }
        }
    }
*/    
    /**
     * Get the package directory used for code generation.
     * 
     * @return directory
     */
    public PackageOrganizer getPackageDirectory() {
        return m_packageDirectory;
    }
    
    /**
     * Run the binding generation using command line parameters.
     * 
     * @param args
     * @throws Exception
     */
    public static void main(String[] args) throws Exception {
        TreeWalker.setLogging(Level.ERROR);
        CodeGenCommandLine parms = new CodeGenCommandLine();
        if (args.length > 0 && parms.processArgs(args)) {
            
            // build set of schemas specified on command line (including via wildcards)
            InsertionOrderedSet fileset = new InsertionOrderedSet();
            URL base = parms.getSchemaRoot();
            File basedir = parms.getSchemaDir();
            SchemaNameFilter filter = new SchemaNameFilter();
            boolean err = false;
            for (Iterator iter = parms.getExtraArgs().iterator(); iter.hasNext();) {
                String path = (String)iter.next();
                int wildstart = path.indexOf('*');
                if (wildstart >= 0) {
                    if (basedir == null) {
                        System.err.println("File name pattern argument not allowed for non-file base: '" + path + '\'');
                        err = true;
                    } else {
                        File dir = basedir;
                        if (File.separatorChar != '/') {
                            path = path.replace('/', File.separatorChar);
                        }
                        int split = path.lastIndexOf(File.separatorChar);
                        String pattern = path;
                        if (split >= 0) {
                            if (wildstart < split) {
                                System.err.println("Wildcard can only be used for file name, not in the directory path, for pattern '" + path + '\'');
                                err = true;
                                continue;
                            } else {
                                dir = new File(basedir, path.substring(0, split));
                                pattern = path.substring(split+1);
                            }
                        }
                        filter.setPattern(pattern);
                        s_logger.debug("Matching file names to command line pattern '" + path + '\'');
                        String[] matches = dir.list(filter);
                        if (matches == null || matches.length == 0) {
                            System.err.println("No files found matching command line pattern '" + pattern +
                                "' in directory " + dir.getAbsolutePath());
                            err = true;
                        } else {
                            for (int i = 0; i < matches.length; i++) {
                                String match = matches[i];
                                fileset.add(new UrlResolver(match, new URL(dir.toURI().toURL(), match)));
                            }
                        }
                    }
                } else {
                    if (basedir == null) {
                        
                        // first try URL as absolute path, then as relative path
                        URL url;
                        try {
                            url = new URL(path);
                        } catch (MalformedURLException e) {
                            url = new URL(base, path);
                        }
                        s_logger.debug("Adding schema URL from command line: " + url.toExternalForm());
                        fileset.add(new UrlResolver(path, url));
                        
                    } else {
                        
                        // first try using path as an absolute URL
                        URL url;
                        try {
                            url = new URL(path);
                        } catch (MalformedURLException e) {
                            
                            // not absolute URL, use as relative path
                            url = new URL(base, path);
                        }
                        s_logger.debug("Adding schema URL from command line: " + url.toExternalForm());
                        fileset.add(new UrlResolver(path, url));
                        
                    }
                }
            }
            if (err) {
                System.exit(2);
            }
            
/*                // add any schemas specified in customizations
                addCustomizedSchemas(base, basedir, parms.getCustomRoot(), fileset);
*/                
            // load the full set of schemas
            CodeGen inst = new CodeGen(parms);
            SchemaElement[] schemas = inst.load(fileset.asList());
            ValidationContext vctx = inst.m_validationContext;
            if (!reportProblems(vctx)) {
                if (inst.customizeSchemas()) {
                    
                    // report on schemas loaded
                    System.out.print("Loaded and validated " + fileset.size() + " specified schema(s)");
                    int refcount = vctx.getSchemaCount() - fileset.size();
                    if (refcount > 0) {
                        System.out.print(" and " + refcount + " referenced schema(s)");
                    }
                    if (parms.isVerbose()) {
                        System.out.println(':');
                        Set topset = new HashSet();
                        for (int i = 0; i < schemas.length; i++) {
                            SchemaElement schema = schemas[i];
                            topset.add(schema);
                            System.out.println(" " + "top-level schema " + schema.getResolver().getName());
                        }
                        for (Iterator iter = vctx.iterateSchemas(); iter.hasNext();) {
                            SchemaElement schema = (SchemaElement)iter.next();
                            if (!topset.contains(schema)) {
                                System.out.println(" " + "referenced schema " + schema.getResolver().getName());
                            }
                        }
                    } else {
                        System.out.println();
                    }
                    
                    // apply the schema customizations and revalidate modified schema
                    inst.applyAndNormalize();
                    inst.pruneDefinitions();
                    inst.validateSchemas(schemas);
                    if (!reportProblems(vctx)) {
                        
                        // generate code and bindings
                        String pack = inst.generate(parms.isVerbose());
                        inst.writeBindings("binding.xml", pack);
                        
                        // print the total number of classes generated
                        int top = 0;
                        int total = 0;
                        List packages = inst.m_packageDirectory.getPackages();
                        for (Iterator iter = packages.iterator(); iter.hasNext();) {
                            PackageHolder hold = (PackageHolder)iter.next();
                            int count = hold.getClassCount();
                            if (count > 0) {
                                int topcount = hold.getTopClassCount();
                                top += topcount;
                                System.out.print("Generated " + topcount + " top-level classes ");
                                if (count != topcount) {
                                    System.out.print("(plus " + (count-topcount) + " inner classes) ");
                                }
                                System.out.println("in package " + hold.getName());
                                total += count;
                            }
                        }
                        if (total == top) {
                            System.out.println("Total classes in model: " + top);
                        } else {
                            System.out.println("Total top-level classes in model: " + top);
                            System.out.println("Total classes (including inner classes) in model: " + total);
                        }
                        
                        // check if dump file to be output
                        File dump = parms.getDumpFile();
                        if (dump != null) {
                            
                            // delete the file if it already exists
                            if (dump.exists()) {
                                dump.delete();
                            }
                            dump.createNewFile();
                            
                            // now print out the class details by package
                            BufferedWriter writer = new BufferedWriter(new FileWriter(dump));
                            DataModelUtils.writeImage(inst.m_packageDirectory, writer);
                            writer.close();
                            
                        }
                    } else {
                        System.err.println("Terminating due to errors in normalized schemas - please report the problem");
                        System.exit(2);
                    }
                } else {
                    System.err.println("Terminating due to errors in customized schemas");
                    System.exit(2);
                }
            } else {
                System.err.println("Terminating due to errors in input schemas");
                System.exit(2);
            }
            
        } else {
            if (args.length > 0) {
                System.err.println("Terminating due to command line or customization errors");
            } else {
                parms.printUsage();
            }
            System.exit(1);
        }
    }

    /**
     * File name pattern matcher.
     */
    private static class SchemaNameFilter implements FilenameFilter
    {
        /** Current match pattern. */
        private String m_pattern;
        
        /**
         * Set the match pattern.
         * 
         * @param pattern
         */
        public void setPattern(String pattern) {
            m_pattern = pattern;
        }
        
        /**
         * Check for file name match.
         * 
         * @param dir
         * @param name
         * @return match flag
         */
        public boolean accept(File dir, String name) {
            boolean match = SchemasetCustom.isPatternMatch(name, m_pattern);
            if (match) {
                s_logger.debug(" matched file name '" + name + '\'');
            }
            return match;
        }
    }
    
    /**
     * Visitor for checking element namespace usage in schema definitions. This just accumulates the set of namespaces
     * used by element definitions.
     */
    private static class SchemaNameVisitor extends SchemaVisitor
    {
        /** Set of namespace URIs used by element definitions. */
        private final Set m_uris;
        
        /**
         * Constructor.
         */
        public SchemaNameVisitor() {
            m_uris = new HashSet();
        }
        
        /**
         * Check if a single namespace is used for all element definitions.
         *
         * @return <code>true</code> if single namespace, <code>false</code> if not
         */
        public boolean isSingleNamespace() {
            return m_uris.size() <= 1;
        }
        
        /**
         * Check if the no-namespace namespace is used by one or more elements.
         *
         * @return <code>true</code> if no-namespace used, <code>false</code> if not
         */
        public boolean isNoNamespaceUsed() {
            return m_uris.contains(null);
        }

        /**
         * Accumulate namespace used by element definition.
         *
         * @param node
         */
        public void exit(ElementElement node) {
            QName qname = node.getQName();
            if (qname != null) {
                m_uris.add(qname.getUri());
            }
            super.exit(node);
        }
    }
}