CGI::AutoForm - Automated abstraction of HTML forms from a data source

 


 use CGI::AutoForm;




 $form = new CGI::AutoForm($dbh,$form_name);

 $form->action($action_url);

 $form->add_group($CGI::AutoForm::EDIT_GROUP,$table_name);

 $form->add_record($current_record);

 $form_html = $self->prepare();

 # insert $form_html into the BODY section of an (X)HTML document via a template




 $group = $form->group_by_name($group_name);

 $bool = $form->validate_query($query,$callback);

 $records = $form->format_query($query);

 $form_copy = $form->clone();

 $form->reset_group();

 


 #
 # an example of customizing a data group's fields...
 #

 $rv = $form->add_group($CGI::AutoForm::INSERT_GROUP,undef,'Vote For Your Favorite Artist','ARTIST_VOTE');

 $fields = $form->db_fields('ARTIST',$CGI::AutoForm::INSERT_GROUP);

 $form->push_field($fields->{ARTIST_NAME});

 $form->add_field( {
    FIELD_NAME => 'VOTE',
    INPUT_CONTROL_TYPE => 'RADIO',
    REQUIRED => 'Y',
    HEADING => 'Vote',
    DATATYPE => 'CHAR',
    INSERTABLE => 'Y',
    },
    [
        { ID => '1', MASK => '*', },
        { ID => '2', MASK => '**', },
        { ID => '3', MASK => '***', },
        { ID => '4', MASK => '****', },
    ]);

 $form_html = $self->prepare( { 'ARTIST_VOTE.ARTIST_NAME' => 'Nonpoint', 'ARTIST_VOTE.VOTE' => 4 } );

  ARTIST
+-------------------------+
| artist_id               |
| artist_name (user_mask) |
| ...                     |
+-------------------------+
          |
          |
          |
  TITLE  /|\
+--------------+
| artist_id    |
| title        |
| year         |
| ...          |
+--------------+

 Form Heading
 <form>
   <div>Data Group1 Heading</div>
   <table>
   tabular view of search results (see below)
    -or-
   vertical view of data group1 fields (see below)
   </table>
   [<div>Data Group2 Heading...</div>
   ...]
 </form>

 for the tabular view of a data group:
 <thead><tr><th>Field1 name</th>[<th>Field2 name</th>...]</tr></thead>
 <tr><td>Value1</td>[<td>Value2</td>...]</tr>
 ...

 for the vertical view of a data group (updatable/insertable groups):
 <tr><td><label>field name</label></td><td>field value/form control</td></tr>
 ...

 $dbh           = $form->dbh();

 $bool          = $form->readonly();
 $name          = $form->name();
 $action        = $form->action();
 $heading       = $form->heading();
 $submit_value  = $form->submit_value();
 $hidden_fields = $form->hidden();
 $meta          = $form->meta();
 $continue      = $form->continue();

 $group_count = $form->add_group($usage[,$table_name,$heading,$group_name,$tabular,$defaults,$mid]);

 $CGI::AutoForm::DISPLAY_ONLY_GROUP
 $CGI::AutoForm::INSERT_GROUP
 $CGI::AutoForm::EDIT_GROUP
 $CGI::AutoForm::SEARCH_GROUP

 Return the new number of groups for $form if successful.
 Return 0 if no fields exist in UI_TABLE_COLUMN or the data dictionary for $table_name.
 Return -1 if a pick_list could not be determined but is required (see C<create_field>).
 Return -2 if $group_name is not unique.
 Return -3 if data dictionary attributes were requested via UI_TABLE_COLUMN but could not be found.
 Return undef if db error.

 $name          = $form->current_group_name();
 $usage         = $form->group_usage();
 $name          = $form->curr_table_name();

 $rec = $form->current_group();

 $group = $form->group_by_name($group_name);

 $form->reset_group();

 $control_type = $form->control_type($field);

 $group_field = $form->create_field($field[,$pick_list,$all_db_defn]);

 $field_count = $form->add_field($field[,$picklist,$mid]);

 $pick_hash = $form->replace_picklist($field_name,$pick_list[,$group]);

 $group_count = $form->push_group($group[,$mid]);

 $field_count = $form->push_field($field[,$mid]);

 $field_count = $form->unshift_field($field);

 $deleted_field = $form->delete_field($field_name[,$group]);

 $fields = $form->db_fields($table_name,$usage);

 Return 0 if no fields exist in UI_TABLE_COLUMN or the data dictionary for $table_name.
 Return -1 if a pick_list could not be determined but is required (see C<create_field>).
 Return -3 if data dictionary attributes were requested via UI_TABLE_COLUMN but could not be found.
 Return undef if db error.

 $html_block = $form->hidden_html();

 $form_html = $self->prepare([$query],[$val_callback],[$rec_callback],[$head_callback]);

 &callback($value,$field,$form)

 &callback($value,$field,$group,$form)

 $group->{html}, a composite of:
 $group->{head_html} (must contain the opening C<table> tag)
 $group->{body_html}
 $group->{tail_html} (must contain the closing C<table> tag)

 $form->{body_html}, a composite of:
 $form->{hidden_html}
 $group->{html} for each group in $form

 $form->{html}, a composite of:
 $form->{head_html} (must contain the opening C<form> tag)
 $form->{body_html}
 $form->{tail_html} (must contain the closing C<form> tag and submit button)

 $form->require_js();

 $field = $form->field_by_name($field_name);

 $fields = $form->field_list();

 $fields = $form->field_hash();

 $fields = $form->next_group();

 $bool = $form->is_readonly([$field]);

 $rec_count = $form->add_record($record[,$group_name]);

 $rec = $form->current_record();

 $bool = $form->validate_query($query[,$callback]);

 ($success,$err_message) = &$callback($value,$field,$group,$form,$query);

 $formatted_query = $form->format_query($query);

 $norm_query = $form->normalize_query($query);

 $struct_query = CGI::AutoForm->structure_query($query);

 $form_copy = $form->clone();

 $form_fields = CGI::AutoForm->extract_query_group($query,$group_name);

 $form_fields = CGI::AutoForm->extract_cut_query_group($query,$group_name);

 $form_fields = CGI::AutoForm->escape($query,$group_name);

 $form_fields = CGI::AutoForm->unescape($query,$group_name);

# AutoForm.pm
#
# $Id: AutoForm.pm,v 1.19 2005/04/11 17:02:35 scottb Exp $
#

package CGI::AutoForm;

use strict;
use DBIx::IO::Mask;
use DBIx::IO::Search;
use DBIx::IO::Table;
use DBIx::IO::GenLib ();


*CGI::AutoForm::VERSION = \'1.03';

*CGI::AutoForm::DISPLAY_ONLY_GROUP = \'DISPLAY ONLY';
*CGI::AutoForm::INSERT_GROUP = \'INSERTABLE';
*CGI::AutoForm::EDIT_GROUP = \'DISPLAY EDIT';
*CGI::AutoForm::SEARCH_GROUP = \'SEARCHABLE';

*CGI::AutoForm::DEFAULT_RADIO_CHECKBOX_COLS = \2;

*CGI::AutoForm::DEFAULT_FIELD_LENGTH = \50;

*CGI::AutoForm::DEFAULT_MULTI_VALUE_DELIMITER = \':';

my @months = ('',qw(
    January
    February
    March
    April
    May
    June
    July
    August
    September
    October
    November
    December
));
my $month_field = {
    FIELD_NAME => '_MONTH',
    INPUT_CONTROL_TYPE => 'SELECT',
    SEARCH_CONTROL_TYPE => 'SELECT',
};
my $month_picklist = [
    map { { ID => $_, MASK => $months[$_] } } ('01'..'12')
];

my $day_field = {
    FIELD_NAME => '_DAY',
    INPUT_CONTROL_TYPE => 'SELECT',
    SEARCH_CONTROL_TYPE => 'SELECT',
};
my $day_picklist = [
    map { { ID => $_, MASK => int($_) } } ('01'..'31')
];

my $year_field = {
    FIELD_NAME => '_YEAR',
    INPUT_CONTROL_TYPE => 'SELECT',
    SEARCH_CONTROL_TYPE => 'SELECT',
};
my $this_year = substr(DBIx::IO::GenLib::local_normal_sysdate(),0,4);

my $hour_field = {
    FIELD_NAME => '_HOUR',
    INPUT_CONTROL_TYPE => 'SELECT',
    SEARCH_CONTROL_TYPE => 'SELECT',
};
my $hour_picklist = [
    map { { ID => $_, MASK => $_ } } ('00'..'23')
];

my $min_field = {
    FIELD_NAME => '_MIN',
    INPUT_CONTROL_TYPE => 'SELECT',
    SEARCH_CONTROL_TYPE => 'SELECT',
};
my $min_picklist = [
    map { { ID => $_, MASK => $_ } } ('00'..'59')
];

my $rel_quantity_field = {
    FIELD_NAME => '_QUANT',
    INPUT_CONTROL_TYPE => 'TEXT',
    SEARCH_CONTROL_TYPE => 'TEXT',
    DATATYPE => 'INT',
    INPUT_SIZE => 4,
    INPUT_MAXLENGTH => 7,
};

my $rel_unit_field = {
    FIELD_NAME => '_UNIT',
    INPUT_CONTROL_TYPE => 'SELECT',
    SEARCH_CONTROL_TYPE => 'SELECT',
};
my $rel_unit_picklist = [
    { ID => 'MINS', MASK => 'Minute(s)' },
    { ID => 'HRS', MASK => 'Hour(s)' },
    { ID => 'DAYS', MASK => 'Day(s)' },
    { ID => 'MONTHS', MASK => 'Month(s)' },
    { ID => 'YEARS', MASK => 'Years(s)' },
];

my $use_range_field = {
    FIELD_NAME => '_UR',
    INPUT_CONTROL_TYPE => 'CHECKBOX',
    SEARCH_CONTROL_TYPE => 'CHECKBOX',
};
my $use_range_picklist = [
    { ID => 1, MASK => 'Check to use this date range criteria' },
];

my $start_range_field = {
    FIELD_NAME => '_RS',
    INPUT_CONTROL_TYPE => 'TEXT',
    SEARCH_CONTROL_TYPE => 'TEXT',
};

my $end_range_field = {
    FIELD_NAME => '_RE',
    INPUT_CONTROL_TYPE => 'TEXT',
    SEARCH_CONTROL_TYPE => 'TEXT',
};

sub new
{
    my ($caller,$dbh,$name) = @_;
    my $class = ref($caller) || $caller;
    my $self = bless({},$class);

    $self->{dbh} = $dbh;
    $self->{curr_group_no} = -1;
    $self->{curr_rec_no} = -1;
    $self->{curr_field_no} = -1;
    $self->{group_hash} = {};
    $self->{group_list} = [];
    $self->name($name);
    $self->{hidden} = {};

    $self->{attr_defn_table_name} = 'ui_table_column';

    return $self;
}

sub dbh
{
    my $self = shift;
    return $self->{dbh};
}

sub readonly
{
    my ($self,$readonly) = @_;
    if (defined($readonly))
    {
        $self->{readonly} = $readonly;
    }
    return $self->{readonly};
}

sub meta
{
    my $self = shift;
    return $self->{meta};
}

sub continue
{
    my $self = shift;
    return $self->{continue};
}

#=pod
#
#=item C<name>
#
#
#Associate a name with $form. Will be used
#in the C<name> attribute of the HTML C<form> element.
#
#=cut
sub name
{
    my ($self,$name) = @_;
    if (defined($name))
    {
        $self->{name} = $name;
    }
    return $self->{name};
}

#=pod
#
#=item C<action>
#
#
#Specify the ACTION attribute of an HTML FORM element, e.g. a URI.
#
#=cut
sub action
{
    my ($self,$action) = @_;
    if (defined($action))
    {
        $self->{action} = $action;
    }
    return $self->{action};
}

#=pod
#
#=item C<heading>
#
#The heading will be displayed near the top of the form in HTML.
#
#=cut
sub heading
{
    my ($self,$heading) = @_;
    if (defined($heading))
    {
        $self->{heading} = $heading;
    }
    return $self->{heading};
}

#=pod
#
#=item C<submit_value>
#
#Override the text that appears in the form's SUBMIT button.
#
#=cut
sub submit_value
{
    my ($self,$submit_value) = @_;
    if (defined($submit_value))
    {
        $self->{submit_value} = $submit_value;
    }
    return $self->{submit_value};
}

##at should rename this add_table_group() then have an add_static_group() for non-db lookups
sub add_group
{
    my ($self,$usage,$table_name,$heading,$group_name,$tabular,$defaults,$mid) = @_;
    my $group = $self->create_group($table_name,$heading,$group_name,$usage,$tabular);
    my $elems = $self->push_group($group,$mid) || return -2;
    return 1 unless $table_name;
    my $fields = $self->field_template($usage,$table_name,$defaults);
    unless ($fields > 0)
    {
        return $fields;
    }
    my ($key,$val);
    while (($key,$val) = each(%$fields))
    {
        $group->{$key} = $val;
    }
    return $elems;
}

sub current_group_name
{
    my ($self) = @_;
    my $group = $self->current_group();
    return $group->{group_name};
}

sub group_usage
{
    my ($self) = @_;
    my $group = $self->current_group();
    return $group->{usage};
}

sub curr_table_name
{
    my ($self) = @_;
    my $group = $self->current_group();
    return $group->{table_name};
}

sub current_group
{
    my ($self) = @_;
    my $i = $self->{curr_group_no};
    $i = 0 if $i < 0;
    return $self->{group_list}[$i];
}

sub group_by_name
{
    my ($self,$group_name) = @_;
##at really need to use Tie::IxHash - should set $self->{group_no} to the index of group_name
    return $self->{group_hash}{$group_name};
}

sub reset_group
{
    my ($self) = @_;
    $self->{curr_group_no} = -1;
    $self->{curr_field_no} = -1;
    $self->{curr_rec_no} = -1;

    return 1;
}

# The VALUE or DISPLAY_VALUE attribute of the next field will be set
# depending on the group usage, after making a copy of the field's hash stored in $group->{field_list}
# the VALUE attribute can be an array ref of values where multiple values are
# allowed.

# Returns either a hashref of the next field's attributes, or
# an array ref of field attributes in the case where a field
# represents a group of form fields (e.g. an insert date field)

# depends on curr_rec_no being incremented - DO NOT USE THIS METHOD WITHOUT next_record method
sub next_field
{
    my ($self) = @_;
    my $field_list = $self->field_list();
    my $field = $field_list->[++$self->{curr_field_no}];
    if (ref($field) && %$field)
    {
        unless ($field = $self->_export_field($field))
        {
            defined($field) || return $field;
            return $self->next_field();
        }
    }
    elsif (ref($field))
    {
        # some kind of bug in perl?? an empty hash ref is an element beyond the number of elements in $field_list?
        undef($field);
    }
    return $field;
}

sub control_type
{
    my ($self,$field,$usage) = @_;
    $usage ||= $self->group_usage();
    return ($usage eq $CGI::AutoForm::SEARCH_GROUP ? $field->{SEARCH_CONTROL_TYPE} : $field->{INPUT_CONTROL_TYPE});
}

# for backwards compatibility
sub _control_type
{
    &control_type;
}

# If $no_group is true, $field->{FORM_ELEMENT_NAME} will not be defined (normally defined
# using the namespace of the current group name). This means you'll have to define it elsewhere.
# $usage must be one of the valid group usage identifiers.
sub create_field
{
    my ($self,$field_attrs,$pick_list,$all_db_defn,$no_group,$usage) = @_;
    $field_attrs = { %$field_attrs };
    $pick_list ||= $field_attrs->{PICK_LIST};
    my $pick_hash;
    my $type = $self->control_type($field_attrs,$usage);
    my $fname = $field_attrs->{FIELD_NAME};

    my $group = $self->current_group();
    $pick_list = [ @{$group->{table}{io}{picklist}{$fname}} ] if (!ref($pick_list) && exists($group->{table}{io}{picklist}{$fname}) && @{$group->{table}{io}{picklist}{$fname}});

    if (($type eq 'SELECT' || $type eq 'RADIO' || $type eq 'CHECKBOX' || $type eq 'CHECKGROUP' || $all_db_defn) && !ref($pick_list))
    {
        my $dbh = $self->dbh();
        my $oldp = $dbh->{PrintError};
        $dbh->{PrintError} = 0;
        my $masker = new DBIx::IO::Mask($self->dbh(),($field_attrs->{ALT_MASK_FIELD} || lc($fname)),
            $field_attrs->{MASK_TABLE_NAME},$field_attrs->{MASK_FIELD_NAME},$field_attrs->{ID_FIELD_NAME},($field_attrs->{NO_CACHE} eq 'Y'));
        $dbh->{PrintError} = $oldp;
        if ($masker)
        {
            $pick_list = $masker->pick_list();
            $pick_hash = $masker->ids_to_mask();
        }
        else
        {
            unless ($all_db_defn)
            {
                if (defined($masker))
                {
                    # may add picklist after creating the field!!                    warn("No mask list found for " . $fname);
                    # return $masker;
                }
                else
                {
                    warn("No mask list found for $field_attrs->{TABLE_NAME}.$fname, perhaps you need to set UI_TABLE_COLUMN.MASK_TABLE_NAME et al appropriately.\n",$dbh->errstr());
                    return $masker;
                }
            }
        }
    }
    elsif (ref($pick_list))
    {
        $pick_hash = { map { $_->{ID} => $_->{MASK} } @$pick_list  };
    }

    if ($all_db_defn && ref($pick_list))
    {
        # if mask test is true and all defn come from db, set SEARCH_MULT_SELECT (will only affect SEARCHABLE groups)
        $field_attrs->{INPUT_CONTROL_TYPE} = 'SELECT';
        $field_attrs->{SEARCH_CONTROL_TYPE} = 'SELECT';
        $field_attrs->{SEARCH_MULT_SELECT} = 3;

        # Special magic for MySQL's SET type
        if ($group->{table}{io}{column_types}{$fname} eq 'SET')
        {
            $field_attrs->{INPUT_CONTROL_TYPE} = 'CHECKGROUP';
            $field_attrs->{SEARCH_CONTROL_TYPE} = 'MATCH TEXT';
            $field_attrs->{SEARCH_MULT_SELECT} = undef;
        }

        $type = $self->control_type($field_attrs,$usage);
    }

    # Special magic for MySQL's SET type
    $field_attrs->{MULTI_INSERT_DELIMITER} = ',' if $group->{table}{io}{column_types}{$fname} eq 'SET';

    $field_attrs->{PICK_LIST} = $pick_list;
    $field_attrs->{PICK_HASH} = $pick_hash;

    return $field_attrs if $no_group;
    
    my $group_name = $self->current_group_name();
    my $elem_name_app;
    if ($type eq 'TEXT' && _isdate($field_attrs) && !$usage)
    {
        # this extension will be stripped via normalize_query()
        # This will signal when a TEXT field needs to be checked that it holds a parseable date string
        $elem_name_app = "._DT";
    }
    elsif ($type eq 'MATCH TEXT')
    {
        # this signals when a search should use wildcards
        $elem_name_app = "._WM";
    }
    elsif ($type eq 'CHECKGROUP')
    {
        $elem_name_app = "._CG";
    }
    $field_attrs->{FORM_ELEMENT_NAME} = "$group_name." . $fname . $elem_name_app;
    return $field_attrs;
}

##at on this and related methods, should check to make sure a group exists!!
sub add_field
{
    my ($self,$field,$picklist,$mid) = @_;
    unless ($field = $self->create_field($field,$picklist))
    {
        return $field;
    }
    return $self->push_field($field,$mid);
}

sub replace_picklist
{
    my ($self,$field_name,$pick_list,$group) = @_;
    $field_name = uc($field_name);
    $group ||= $self->current_group();
    $group->{field_hash}{$field_name}{PICK_LIST} = $pick_list;
    my $pick_hash = { map { $_->{ID} => $_->{MASK} } @$pick_list  };
    return $group->{field_hash}{$field_name}{PICK_HASH} = $pick_hash;
}

sub push_group
{
    my ($self,$group,$mid) = @_;
    defined($mid) || ($mid = @{$self->{group_list}});
    my $name = $group->{group_name};
    exists($self->{group_hash}{$name}) && (warn("\$group_name [$name] not unique"),return undef);
    $self->{group_hash}{$name} = $group;
    splice(@{$self->{group_list}},$mid,0,$group);
    my $elems = @{$self->{group_list}};
    $self->{curr_group_no} = $mid;
    return $elems;
}

sub push_field
{
    my ($self,$field,$mid) = @_;
    my $field_list = $self->field_list();
    defined($mid) || ($mid = @{$field_list});
    my $field_hash = $self->field_hash();
    $field_hash->{$field->{FIELD_NAME}} = $field;
    splice(@{$field_list},$mid,0,$field);
    return scalar(@{$field_list});
}


##at would be nice to have a field {NEVER_SHOW} that flagged if it should be included in the record at all (hidden or otherwise)

sub unshift_field
{
    my ($self,$field) = @_;
    my $field_list = $self->field_list();
    my $field_hash = $self->field_hash();
    $field_hash->{$field->{FIELD_NAME}} = $field;
    return unshift(@{$field_list},$field);
}

sub pop_field
{
    my ($self) = @_;
    my $field_list = $self->field_list();
    my $field_hash = $self->field_hash();
    my $field = pop(@{$field_list});
    delete($field_hash->{$field->{FIELD_NAME}}) if ref($field);
    return $field;
}

sub shift_field
{
    my ($self) = @_;
    my $field_list = $self->field_list();
    my $field_hash = $self->field_hash();
    my $field = shift(@{$field_list});
    delete($field_hash->{$field->{FIELD_NAME}}) if ref($field);
    return $field;
}

# Assumes that FIELD_NAME is unique
# Modifies by reference (no copying is done)
sub delete_field
{
    my ($self,$field_name,$group) = @_;
    $field_name = uc($field_name);
    my $field_list;
    my $field_hash;
    if ($group)
    {
        $field_list = $group->{field_list};
        $field_hash = $group->{field_hash};
    }
    else
    {
        $field_list = $self->field_list();
        $field_hash = $self->field_hash();
    }
    my $i = 0;
    foreach my $field (@{$field_list})
    {
        next if $field->{FIELD_NAME} eq $field_name;
        $field_list->[$i] = $field;
        $i++;
    }
    return if @{$field_list} == $i;
    pop(@{$field_list});
    return delete($field_hash->{$field_name});
}

sub db_fields
{
    my ($self,$table_name,$usage) = @_;
    my $fields = $self->field_template($usage,$table_name,undef,1);
    unless ($fields > 0)
    {
        return $fields;
    }
    return $fields->{field_hash};
}

# By convention, all TABLE_NAME GROUP_NAME and FIELD_NAME are case-sensitive and always UPPER CASE

# If successful, return a hashref where:
# field_list => fields in the order they should appear on the form
# field_hash => fields keyed by UI_TABLE_COLUMN.FIELD_NAME
# fields may be defined in UI_TABLE_COLUMN or may be derived from the db dict via table_name

# Return 0 if no fields exist in UI_TABLE_COLUMN for $table_name
# Return -1 if a pick_list could not be culled using DBIx::IO::Mask
# Return -3 if data dictionary attributes were requested but could not be found
# Return undef if db error

# $defaults override those listed in the db
##at maybe cache field_list for the entire class??
sub field_template
{
    my ($self,$usage,$table_name,$defaults,$no_group) = @_;
    $defaults ||= {};
    my $orig_table_name = $table_name;
    $table_name = uc($table_name);

    my $group = $self->current_group();

    my $recs = [];
    my $searcher = new DBIx::IO::Search($self->dbh(),$self->{attr_defn_table_name});
    if (ref($searcher))
    {
        $searcher->build_scalar_crit('TABLE_NAME','=',$table_name);
##at    $searcher->build_scalar_crit($usage,'=','Y') unless $usage eq $CGI::AutoForm::DISPLAY_ONLY_GROUP || $usage eq $CGI::AutoForm::EDIT_GROUP;
        $recs = $searcher->search(undef,[ 'APPEAR_ORDER' ]) || return undef;
    }
    elsif (!defined($searcher))
    {
        return undef;
    }

    my $all_db_defn;
    my $table;
    unless (@$recs)
    {
        # Try and see if a table exists
        if ($table = new DBIx::IO::Table($self->dbh(),undef,undef,$orig_table_name))
        {
            foreach my $col (@{$table->{io}{col_list}})
            {
                # Ignore LOB types completely (Oracle)
                push(@$recs, { TABLE_NAME => $table_name, FIELD_NAME => $col, UPDATABLE => 'Y', SEARCHABLE => 'Y', INSERTABLE => 'Y', })
                    unless $table->{io}->is_ignore_type($table->column_type($col));
            }
            unless ($no_group)
            {
                $group->{table} = $table;
            }
            $all_db_defn++;
        }
        else
        {
            defined($table) || return $table;
        }
        @$recs || (warn("No fields defined for $orig_table_name"),return 0);
    }
    my $field_hash = {};
    my $fields = [];
    foreach my $rec (@$recs)
    {
        my $fname = uc($rec->{FIELD_NAME});
        $rec->{FIELD_NAME} = $fname;
        if ($rec->{USE_DATA_DICT} eq 'Y' || $all_db_defn)
        {
            my $rv;
            unless ($rv = $self->use_data_dict($rec,$orig_table_name,$table,$no_group))
            {
                defined($rv) && return -3;
                return $rv;
            }
        }
        else
        {
            $rec->{DATATYPE} = 'CHAR' unless $rec->{DATATYPE};
            $rec->{INPUT_SIZE} ||= $CGI::AutoForm::DEFAULT_FIELD_LENGTH;
            $rec->{INPUT_MAXLENGTH} ||= $CGI::AutoForm::DEFAULT_FIELD_LENGTH;
        }
        if (!$rec->{SEARCH_CONTROL_TYPE} && $usage eq $CGI::AutoForm::SEARCH_GROUP)
        {
            my $ir = $rec->{INPUT_CONTROL_TYPE};
            if ($ir)
            {
                $rec->{SEARCH_CONTROL_TYPE} = 'DATERANGE' if $ir eq 'DATE';
                $rec->{SEARCH_CONTROL_TYPE} = 'DATETRANGE' if $ir eq 'DATETIME';
                $rec->{SEARCH_CONTROL_TYPE} = 'SELECT' if $ir eq 'SELECT';
                $rec->{SEARCH_CONTROL_TYPE} = 'RADIO' if $ir eq 'RADIO';
            }
            elsif (_isdate($rec))
            {
                $rec->{SEARCH_CONTROL_TYPE} = 'DATETRANGE' if $rec->{DATATYPE} eq 'DATETIME';
                $rec->{SEARCH_CONTROL_TYPE} = 'DATERANGE' if $rec->{DATATYPE} eq 'DATE';
            }
            else
            {
                $rec->{SEARCH_CONTROL_TYPE} = 'TEXT';
            }
        }
        elsif (!$rec->{INPUT_CONTROL_TYPE} && !$self->is_readonly($rec))
        {
            if (_isdate($rec))
            {
                $rec->{INPUT_CONTROL_TYPE} = 'DATETIME' if $rec->{DATATYPE} eq 'DATETIME';
                $rec->{INPUT_CONTROL_TYPE} = 'DATE' if $rec->{DATATYPE} eq 'DATE';
            }
            else
            {
                $rec->{INPUT_CONTROL_TYPE} = 'TEXT';
            }
        }

        # Disable searches for Oracle LONG or RAW types (they will error)
        my $type = '';
        $rec->{SEARCHABLE} = 'N'
            if $group->{table} && ($type = $group->{table}->column_type($rec->{FIELD_NAME})) && $group->{table}{ioclass} eq 'DBIx::IO::OracleIO' &&
                ($type eq 'LONG' || $type eq 'RAW' || $type eq 'LONG RAW');

        # the required field, therefore only applies to a non-searchable group (exept as noted above where you can set it explicitly)
        # this needs clearer docs: the REQUIRED field DOES NOT APPLY TO groups that are designated as $CGI::AutoForm::SEARCH_GROUP, the REQUIRED field is unset (undef())
        # you can, however explicitly set $group->{REQUIRED} = 'Y' after calling add_group()
        undef($rec->{REQUIRED}) if $usage eq $CGI::AutoForm::SEARCH_GROUP;
        $rec->{DEFAULT_VALUE} = $defaults->{$fname} if defined($defaults->{$fname});
        $rec->{HEADING} = $fname unless length($rec->{HEADING});
        $rec->{BRIEF_HEADING} = $fname if !length($rec->{BRIEF_HEADING}) && $all_db_defn;
        my $field = $self->create_field($rec,undef,$all_db_defn,$no_group,$usage);
        unless ($field)
        {
            defined($field) && return -1;
            return $field;
        }
        push(@$fields,$field);
        $field_hash->{$fname} = $field;
    }
    return { field_list => $fields, field_hash => $field_hash };
}

##at because intrinsic date types can differ from explicit ones, this should always be executed if a table exists and tableIO is performed

# Return 0 if FIELD_NAME does not exist in table
sub use_data_dict
{
    my ($self,$field,$table_name,$table,$no_group) = @_;
    unless ($table && %$table)
    {
        unless ($table = new DBIx::IO::Table($self->dbh(),undef,undef,$table_name))
        {
            return $table;
        }
        unless ($no_group)
        {
            my $group = $self->current_group();
            $group->{table} = $table;
        }
    }
    my $io = $table->{io};
    my $fname = $field->{FIELD_NAME};
    return 0 unless $io->column_type($fname);
    if ($io->is_datetime($fname))
    {
        $field->{_imp_datetype} = 'DATETIME';
    }
    elsif ($io->is_date($fname))
    {
        $field->{_imp_datetype} = 'DATE';
    }
    unless ($field->{DATATYPE})
    {
        $field->{DATATYPE} = $field->{_imp_datetype} if $field->{_imp_datetype};
    }
    my $dval = $io->default_value($fname);
    # Dates cannot have default values from the data dictionary, I don't want to bother with formatting them!
    undef($dval) if $field->{_imp_datetype};
    $field->{DEFAULT_VALUE} = $dval unless length($field->{DEFAULT_VALUE}) || !length($dval);
    $field->{REQUIRED} = ($io->required($fname) ? 'Y' : undef) unless $field->{REQUIRED};
    my $flen = $io->field_length($fname);
    if ($field->{DATATYPE} eq 'DATE')
    {
##at const depends on output date format
        $flen = 11;
    }
    elsif ($field->{DATATYPE} eq 'DATETIME')
    {
##at const depends on output date format
        $flen = 17;
    }
    $field->{INPUT_SIZE} = ($flen > $CGI::AutoForm::DEFAULT_FIELD_LENGTH ? $CGI::AutoForm::DEFAULT_FIELD_LENGTH : $flen)
        unless $field->{INPUT_SIZE};
    $field->{INPUT_MAXLENGTH} = $flen unless $field->{INPUT_MAXLENGTH};
    return $field;
}

# Set the html attribute of $field with, and return a string of HTML
# according to INPUT_CONTROL_TYPE.
# Note that INPUT_CONTROL_TYPE may have been set by _export_field() depending
# on the way $field will be used in the form.
sub field_html
{
    my ($self,$field) = @_;
    my $type = $self->control_type($field);
    if ($type eq 'TEXT' || $type eq 'PASSWORD' || $type eq 'MATCH TEXT')
    {
        return ($field->{html} = $self->text_pass_html($field,$type));
    }
    elsif ($type eq 'TEXTAREA')
    {
        return ($field->{html} = $self->textarea_html($field));
    }
    elsif ($type eq 'SELECT')
    {
        return ($field->{html} = $self->select_html($field));
    }
    elsif ($type eq $CGI::AutoForm::DISPLAY_ONLY_GROUP)
    {
        return ($field->{html} = $self->display_html($field));
    }
    elsif ($type eq 'CHECKBOX' or $type eq 'CHECKGROUP')
    {
        return ($field->{html} = $self->checkbox_html($field));
    }
    elsif ($type eq 'RADIO')
    {
        return ($field->{html} = $self->radio_html($field));
    }
    elsif ($type eq 'FILE')
    {
        return ($field->{html} = $self->fileupload_html($field));
    }
    else
    {
        die("Field type [$type] not recognized for $field->{FIELD_NAME}");
        return undef;
    }
}

# File type input.
##at not sure if the comments below are correct?
## In order for file uploads to work on a form,
## the enctype="file" attribute needs to be added to the form tag.
sub fileupload_html
{
    my ($self,$field) = @_;

    my $need_req = $field->{REQUIRED} eq 'Y';
    my $need_req_class = 0;
    if ($need_req)
    {
        $need_req_class = 1 unless $field->{ELEMENT_ATTRS} =~ 
            s/CLASS\s*=\s*['"]?(.*?)['"]?/CLASS="$1 REQI"/i;
    }

    my $name = qq[name="] . $self->escape($field->{FORM_ELEMENT_NAME}) . qq["];
    my $class = $need_req_class ? qq[CLASS="REQI" ] : "";
    my $attrs = $field->{ELEMENT_ATTRS};

    my $html = qq[<input type="file" $class $name $attrs>];

    $self->{form_enctype_attr} = qq[enctype="multipart/form-data"];

    return $html;
}

# A checkbox field results in a series of checkboxes, one for each
# PICK_LIST element, multiple values may be checked for the same field.
# A PICK_LIST attribute of $field is necessary (see create_field())

# The default layout of the checkboxes will be a tabular with
# $CGI::AutoForm::DEFAULT_RADIO_CHECKBOX_COLS number of rows, this can
# be overridden by specifying UI_TABLE_COLUMN.RADIO_CHECKBOX_COLS
sub checkbox_html
{
    my ($self,$field) = @_;

    my %checked;
    if (ref($field->{VALUE}) eq 'ARRAY')
    {
        map { $checked{$_} = 'CHECKED ' } (@{$field->{VALUE}});
    }
    elsif (!ref($field->{VALUE}))
    {
        $checked{$field->{VALUE}} = 'CHECKED ' if defined($field->{VALUE});
    }
    else
    {
        die "A horrible death";
    }

    my $need_req = $field->{REQUIRED} eq 'Y';
    my $need_req_class = 0;
    if ($need_req)
    {
        $need_req_class = 1 unless $field->{ELEMENT_ATTRS} =~ s/CLASS\s*=\s*['"]?(.*?)['"]?/CLASS="$1 REQI"/i;
    }

##at should have a {RADIO_CHECKBOX_COLS} = 0 where no <TABLE> and related tags are generated
    my $ret = '<TABLE><TR>';
    my $i;
    my $cols = ($field->{RADIO_CHECKBOX_COLS} ? $field->{RADIO_CHECKBOX_COLS} : $CGI::AutoForm::DEFAULT_RADIO_CHECKBOX_COLS);
    foreach my $opt (@{$field->{PICK_LIST}})
    {
        my $new_row = $i && !($i % $cols);
        $new_row && ($ret .= '</TR><TR>');

        $ret .= '<TD><INPUT TYPE="CHECKBOX" ' .
        qq[NAME="] . $self->escape($field->{FORM_ELEMENT_NAME}) . '" ' .
        qq[VALUE="] . $self->escape($opt->{ID}) . '" ' .
        $checked{$opt->{ID}} .
        ($need_req_class ? qq[CLASS="REQI" ] : "") .
        "$field->{ELEMENT_ATTRS}>" .
        $self->escape($opt->{MASK}) . '</TD>';
        $i++;
    }
    $ret .= '</TR></TABLE>';
    return $ret;
}

# A radio field results in an HTML radio control group
# where each element can be toggled but only 1 element
# in the group can be in the on position. This differs from
# a checkbox group in that the field can have only 1 value.
# Choices for the field value are taken from the field's PICK_LIST;
# a PICK_LIST attribute of $field must exist (see create_field())

# Refer to checkbox_html() for a description of the layout of the
# radio group.
sub radio_html
{
    my ($self,$field) = @_;
    my $val = $field->{VALUE};
    ref($val) && (die("Radio groups cannot be muti-valued"),return undef);
    my %checked;
    $checked{$val} = 'CHECKED ' if defined($val);
    my $ret = '<TABLE><TR>';
    my $i;
    my $cols = ($field->{RADIO_CHECKBOX_COLS} ? $field->{RADIO_CHECKBOX_COLS} : $CGI::AutoForm::DEFAULT_RADIO_CHECKBOX_COLS);

    my $need_req = $field->{REQUIRED} eq 'Y';
    my $need_req_class = 0;
    if ($need_req)
    {
        $need_req_class = 1 unless $field->{ELEMENT_ATTRS} =~ s/CLASS\s*=\s*['"]?(.*?)['"]?/CLASS="$1 REQI"/i;
    }

    foreach my $opt (@{$field->{PICK_LIST}})
    {
        my $new_row = $i && !($i % $cols);
        $new_row && ($ret .= '</TR><TR>');

        $ret .= '<TD><INPUT TYPE="RADIO" ' .
        qq[NAME="] . $self->escape($field->{FORM_ELEMENT_NAME}) . '" ' .
        qq[VALUE="] . $self->escape($opt->{ID}) . '" ' .
        $checked{$opt->{ID}} .
        ($need_req_class ? qq[CLASS="REQI" ] : "") .
        "$field->{ELEMENT_ATTRS}>" .
        $self->escape($opt->{MASK}) . '</TD>';
        $i++;
    }
    $ret .= '</TR></TABLE>';
    return $ret;
}

##at new field_group of 'COMBO' would be nice to accept free-form or a select list

sub display_html
{
    my ($self,$field) = @_;
    my $val = $field->{DISPLAY_VALUE};
    my $dval = '';
    if (ref($val))
    {
        foreach my $v (@$val)
        {
            $dval .= $self->escape($val) . '<BR>';
        }
        chop($dval);
        chop($dval);
        chop($dval);
        chop($dval);
    }
    else
    {
        $dval = $self->escape($val);
        $dval =~ s/\r?\n/<BR>/gs;
    }
    return $dval;
}

sub select_html
{
    my ($self,$field) = @_;

    my $need_req = $field->{REQUIRED} eq 'Y';
    my $need_req_class = 0;
    if ($need_req)
    {
        $need_req_class = 1 unless $field->{ELEMENT_ATTRS} =~ s/CLASS\s*=\s*['"]?(.*?)['"]?/CLASS="$1 REQI"/i;
    }

    my $ret = '<SELECT ' .
        qq[NAME="] . $self->escape($field->{FORM_ELEMENT_NAME}) . '" ' .
        "$field->{ELEMENT_ATTRS} " .
        ($need_req_class ? qq[CLASS="REQI" ] : "");

    my %selected;
    if (ref($field->{VALUE}) eq 'ARRAY')
    {
        map { $selected{$_} = 'SELECTED ' } (@{$field->{VALUE}});
    }
    elsif (!ref($field->{VALUE}))
    {
        $selected{$field->{VALUE}} = 'SELECTED ' if defined($field->{VALUE});
    }
    else
    {
        die "The value of $field->{FIELD_NAME} must be an array ref or scalar, found [$field->{VALUE}]";
    }

    my $usage = $self->group_usage();
    if ($usage eq $CGI::AutoForm::SEARCH_GROUP)
    {
        $ret .= ($field->{SEARCH_MULT_SELECT} ? qq[MULTIPLE SIZE="$field->{SEARCH_MULT_SELECT}" ] : '');
    }
    $ret .= '>';
    unless ($need_req)
    {
        $ret .= '<OPTION VALUE="" ' . (%selected ? '' : 'SELECTED ') . '>&nbsp;</OPTION>';
    }

    foreach my $opt (@{$field->{PICK_LIST}})
    {
        $ret .= qq[<OPTION VALUE="] . $self->escape($opt->{ID}) . qq[" $selected{$opt->{ID}}>] . $self->escape($opt->{MASK}) . '</OPTION>';
    }
    $ret .= '</SELECT>';
    return $ret;
}

sub text_pass_html
{
    my ($self,$field,$type) = @_;

    my $need_req = $field->{REQUIRED} eq 'Y';
    my $need_req_class = 0;
    if ($need_req)
    {
        $need_req_class = 1 unless $field->{ELEMENT_ATTRS} =~ s/CLASS\s*=\s*['"]?(.*?)['"]?/CLASS="$1 REQI"/i;
    }

    my $val = $field->{VALUE};
    $val = '**INVALID REFERENCE**' if ref($val);

    $self->{_ast_match}++ if $type eq 'MATCH TEXT';

##at $type = 'TEXT' if $type eq 'MATCH TEXT' ???
    return qq[<INPUT TYPE="$type" ] .
        qq[NAME="] . $self->escape($field->{FORM_ELEMENT_NAME}) . '" ' .
        qq[VALUE="] . $self->escape($val) . '" ' .
        ($field->{INPUT_SIZE} ? qq[SIZE="$field->{INPUT_SIZE}" ] : '') .
        ($field->{INPUT_MAXLENGTH} ? qq[MAXLENGTH="$field->{INPUT_MAXLENGTH}" ] : '') .
        ($need_req_class ? qq[CLASS="REQI" ] : "") .
        "$field->{ELEMENT_ATTRS}>" .
        ($type eq 'MATCH TEXT' ? '**' : '');
}

sub textarea_html
{
    my ($self,$field) = @_;

    my $need_req = $field->{REQUIRED} eq 'Y';
    my $need_req_class = 0;
    if ($need_req)
    {
        $need_req_class = 1 unless $field->{ELEMENT_ATTRS} =~ s/CLASS\s*=\s*['"]?(.*?)['"]?/CLASS="$1 REQI"/i;
    }

    my $val = $field->{VALUE};
    $val = '**INVALID REFERENCE**' if ref($val);

    return '<TEXTAREA ' .
        qq[NAME="] . $self->escape($field->{FORM_ELEMENT_NAME}) . '" ' .
        ($need_req_class ? qq[CLASS="REQI" ] : "") .
        "$field->{ELEMENT_ATTRS}>" .
        $self->escape($val,1) .
        '</TEXTAREA>';
}

sub hidden
{
    my ($self,$hidden) = @_;
    if (defined($hidden))
    {
        $self->{hidden} = $hidden;
    }
    return $self->{hidden};
}

sub hidden_html
{
    my ($self) = @_;
    my ($name,$val);
    my $ret;
    while (($name,$val) = each(%{$self->{hidden}}))
    {
        if (!ref($val))
        {
            $val = [ $val ];
        }
        elsif (!(ref($val) eq 'ARRAY'))
        {
            die "The hidden field [$name] must be an array ref or scalar, found [$val]";
        }

        foreach my $subval (@$val)
        {
            $ret .= qq[<INPUT TYPE="HIDDEN" NAME="] . $self->escape($name) . '" VALUE="' . $self->escape($subval) . '">';
        }
    }
    $self->{hidden_html} = $ret;
    return $ret;
}

# create a copy of $field processing and return the new $field
sub _export_field
{
    my ($self,$field) = @_;
    $field = { %$field };
    my $usage = $self->group_usage();

    # First check if its read-only at the record level
    # because an updatable record might have a read-only field
    # which will be handled differently in the updatable context
    if ($self->is_readonly())
    {
        return $self->_export_ro($field);
    }
    elsif ($usage eq $CGI::AutoForm::EDIT_GROUP)
    {
        return $self->_export_update($field);
    }
    elsif ($usage eq $CGI::AutoForm::SEARCH_GROUP)
    {
        return $self->_export_search($field);
    }
    elsif ($usage eq $CGI::AutoForm::INSERT_GROUP)
    {
        return $self->_export_insert($field);
    }
    else
    {
        die("Group type [$usage] not recognized");
    }
}

# add to hidden a record from {data} if it exists
sub _export_update
{
    my ($self,$field) = @_;
    my ($rec,$val);
    my $isdate = _isdate($field);
    my $fname = $field->{FIELD_NAME};
    my $gname = $self->current_group_name();
    $rec = $self->{struct_query}{$gname};

# may have empty records if no fields in the record require a value
#    unless (($rec = $self->{struct_query}{$gname}) && ref($rec))
#    {
#        die("No record exists for field: " . $fname);
#        return undef;
#    }

    if ($self->is_readonly($field))
    {
        $self->_assign_ro_val($field,$rec->{$fname},$isdate);
    }
    else
    {
        $val = $self->_extract_query_val($fname);
        if ($self->_isdate_inscontrol($field))
        {
            return $self->expand_date_fields($field,$val);
        }
        else
        {
            if ($field->{INPUT_CONTROL_TYPE} eq 'CHECKGROUP')
            {
                $val = [ split($field->{MULTI_INSERT_DELIMITER}, $val) ];
            }
            $field->{VALUE} = $val;
        }
    }
    my $group = $self->current_group();
    push(@{$group->{export_rec}[$self->{curr_rec_no}]},$field);
    return $field;
}

##at a query must only be introduced through prepare() in order for it to be cleaned,
##at structured and recognized properly in next_field()

# some elements of hidden may be deleted if _extract_query_val() is called with $delete_end = TRUE (currently only true if 'INSERTABLE')

# delete query element(s) and if last field in rec, delete all query elements of that group
sub _extract_query_val
{
    my ($self,$fname,$delete_end) = @_;
    my ($rec,$val);
##at for updates, $form_pre must match $fen exactly for this to work
    my $form_pre = $self->current_group_name() . '.' . $fname;
    my $hidden = $self->hidden();
    $val = $hidden->{$form_pre};

    delete($hidden->{$form_pre});
    if ($delete_end)
    {
        my $field_list = $self->field_list();
        $self->delete_hidden_group() if ($self->{curr_field_no} == $#$field_list);
    }
    return $val;
}

sub delete_hidden_group
{
    my ($self) = @_;
    my $hidden = $self->hidden();
    my $name = $self->current_group_name();
    $name = quotemeta($name);
    foreach my $field_name (keys(%$hidden))
    {
        delete($hidden->{$field_name}) if $field_name =~ /^$name\./;
    }
    return $hidden
}

##at should be able to give an insert group default values and have some of those default values
##at be non-insertable but simply informational so that the user knows what its value will be
##at and can't change it this will require an insert group supporting readonly fields

sub _export_insert
{
    my ($self,$field) = @_;
    return 0 unless $field->{INSERTABLE} eq 'Y';
    my ($rec,$val);
    if (exists($self->{struct_query}{$self->current_group_name()}))
    {
        $val = $self->_extract_query_val($field->{FIELD_NAME},1);
    }
    elsif ($rec = $self->current_record())
    {
        $val = $rec->{$field->{FIELD_NAME}};
        $val = $field->{DEFAULT_VALUE} if !length($val) && $self->{coalesce_default_insert};
    }
    else
    {
        $val = $field->{DEFAULT_VALUE};
    }

    if ($self->_isdate_inscontrol($field))
    {
        return $self->expand_date_fields($field,$val);
    }
    elsif ($field->{FIELD_GROUP} eq 'CONFIRM')
    {
        # Only for inserts
        return $self->expand_confirm_fields($field);
    }
    if ($field->{INPUT_CONTROL_TYPE} eq 'CHECKGROUP')
    {
        $val = [ split($field->{MULTI_INSERT_DELIMITER}, $val) ];
    }
    $field->{VALUE} = $val;
    my $group = $self->current_group();
    push(@{$group->{export_rec}[$self->{curr_rec_no}]},$field);
    return $field;
}

# Only for inserts
sub expand_confirm_fields
{
    my ($self,$field) = @_;
    my $new_field = { %$field };
    $new_field->{FIELD_NAME} = '_CONFIRM1';
    my $conf = _copy_field($new_field,$field);
    $conf = $self->create_field($conf);
    my $group = $self->current_group();
    push(@{$group->{export_rec}[$self->{curr_rec_no}]},$conf);
    $new_field->{FIELD_NAME} = '_CONFIRM2';
    my $conf2 = _copy_field($new_field,$field);
    $conf2 = $self->create_field($conf2);
    $conf2->{HEADING} = 'Re-enter';
    push(@{$group->{export_rec}[$self->{curr_rec_no}]},$conf2);

    return [ $conf,$conf2 ];
}

sub _isdate
{
    my ($field) = @_;
    return ($field->{DATATYPE} eq $DBIx::IO::GenLib::DATE_TYPE || $field->{DATATYPE} eq $DBIx::IO::GenLib::DATETIME_TYPE);
}

sub _isdate_inscontrol
{
    my ($self,$field) = @_;
    my $type = $self->control_type($field);
    return ($type eq 'DATE' || $type eq 'DATETIME');
}

sub _export_search
{
    my ($self,$field) = @_;
    return 0 unless $field->{SEARCHABLE} eq 'Y';
    my $qhit;
    my $gname = $self->current_group_name();
    my $fname = $field->{FIELD_NAME};
    $qhit++ if (exists($self->{struct_query}{$gname}));
    my $rec = $self->current_record();
    ref($rec) && (die("Data record exists for search crit"),return undef);
    my $type = $field->{SEARCH_CONTROL_TYPE};
    if ($type eq 'DATERANGE' || $type eq 'DATETRANGE')
    {
        my ($ur,$rs,$re);
        if ($qhit)
        {
            $ur = $self->_extract_query_val("$fname._UR");
            $rs = $self->_extract_query_val("$fname._RS");
            $re = $self->_extract_query_val("$fname._RE",1);
        }
        return $self->expand_date_search_fields($field,$ur,$rs,$re);
    }
    elsif ($type eq 'DATE' || $type eq 'DATETIME')
    {
        my $dval = '_SYSDATE';
        $dval = $self->_extract_query_val($fname,1) if ($qhit);
        return $self->expand_date_fields($field,$dval);
    }
    elsif ($type eq 'RANGE')
    {
        my ($rs,$re);
        if ($qhit)
        {
            $rs = $self->_extract_query_val("$fname._RS");
            $re = $self->_extract_query_val("$fname._RE",1);
        }
        return $self->expand_range_fields($field,$rs,$re);
    }
    elsif ($type eq 'MATCH TEXT')
    {
        $field->{VALUE} = $self->_extract_query_val("$fname._WM",1) if ($qhit);
    }
    else
    {
        $field->{VALUE} = $self->_extract_query_val($fname,1) if ($qhit);
    }
    my $group = $self->current_group();
    push(@{$group->{export_rec}[$self->{curr_rec_no}]},$field);
    return $field;
}

sub expand_range_fields
{
    my ($self,$field,$rs,$re) = @_;
    my @fields;
    my $startf = _copy_field($start_range_field,$field);
    $startf = $self->create_field($startf);
    $startf->{VALUE} = $rs;
    push(@fields,$startf);
    my $endf = _copy_field($end_range_field,$field);
    $endf = $self->create_field($endf);
    $endf->{VALUE} = $re;
    push(@fields,$endf);
    my $group = $self->current_group();
    push(@{$group->{export_rec}[$self->{curr_rec_no}]},@fields);
    return \@fields;
}

# set the read-only flag according to form -> group -> if read-only, do not transmit form element properties
sub _export_ro
{
    my ($self,$field) = @_;
    my ($rec,$val);
    my $isdate = _isdate($field);

    my $usage = $self->group_usage();
    my $group = $self->current_group();

    # Check $group->{data} BEFORE any {hidden} fields because an incomplete record may be passed
    # around in the hidden fields but display of the full record is normally desired
    # e.g. passing around the record's PK is useful but the full record should be displayed
    if ($rec = $self->current_record())
    {
    }
    elsif (exists($self->{struct_query}{$self->current_group_name()}))
    {
        $rec = $self->{struct_query}{$self->current_group_name()};
    }

    $rec = {} unless $rec;

    # If the group is being used for display, then display only those values contained in $rec
    # If the original group usage is other than DISPLAY ONLY, we want to show all user input, and lack of input
    # e.g. if we're conducting a session and are showing the summary screen, any fields left blank by the user should be revealed
    # unless the field is empty and the user is not allowed to enter a value for the field
    exists($rec->{$field->{FIELD_NAME}}) || return 0 if ($usage eq $CGI::AutoForm::DISPLAY_ONLY_GROUP && !$group->{tabular}) ||
        ($usage eq $CGI::AutoForm::INSERT_GROUP && $field->{INSERTABLE} ne 'Y') ||
        ($usage eq $CGI::AutoForm::EDIT_GROUP && $field->{UPDATABLE} ne 'Y') ||
        ($usage eq $CGI::AutoForm::SEARCH_GROUP && $field->{SEARCHABLE} ne 'Y');

    $val = $rec->{$field->{FIELD_NAME}};

    $self->_assign_ro_val($field,$val,$isdate);
    push(@{$group->{export_rec}[$self->{curr_rec_no}]},$field);
    return $field;
}

sub _assign_ro_val
{
    my ($self,$field,$val,$isdate) = @_;
    $val = _display_normal_date($val,$field->{DATATYPE}) if ($isdate);

    if ($field->{INPUT_CONTROL_TYPE} eq 'CHECKGROUP' && ref($field->{PICK_HASH}))
    {
        foreach my $sp (split($field->{MULTI_INSERT_DELIMITER},$val))
        {
            $field->{DISPLAY_VALUE} .= (exists($field->{PICK_HASH}{$sp}) ? $field->{PICK_HASH}{$sp} : 'INVALID!') . ', ';
        }
        chop($field->{DISPLAY_VALUE});
        chop($field->{DISPLAY_VALUE});
    }
    else
    {
        $field->{DISPLAY_VALUE} = (ref($field->{PICK_HASH}) ? $field->{PICK_HASH}{$val} : $val);
    }

    $field->{INPUT_CONTROL_TYPE} = $CGI::AutoForm::DISPLAY_ONLY_GROUP;
    delete($field->{FORM_ELEMENT_NAME});
}

sub _display_normal_date
{
    my ($dateval,$type) = @_;
    my ($year,$mon,$day,$hr,$min,$sec) = $dateval =~ /(\d\d\d\d)(\d\d)(\d\d)(\d\d)?(\d\d)?(\d\d)?/;
    my $date = $day . '-' . substr($months[$mon],0,3) . '-' . $year;
    if ($type eq $DBIx::IO::GenLib::DATE_TYPE)
    {
        return $date;
    }
    elsif ($type eq $DBIx::IO::GenLib::DATETIME_TYPE)
    {
        return "$date $hr:$min";
    }
    else
    {
        die("Invalid date type: $type");
        return undef;
    }
}

##at how about a group_callback???
##at default head and tail html (and other default HTML) should be available as a constant
# populate export attr of each group and integrate query into hidden fields
# the query passed in will be structured, date compressed, and validated (see structure_query(),validate_query())
#
# Advanced:
# All $query values will be also added to the form's hidden attribute, except if they
# have been used to override values in the current form (we don't want to confuse these with data
# submitted by the operator).

sub prepare
{
    my ($self,$query,$val_callback,$rec_callback,$head_callback) = @_;
    $self->{val_callback} = $val_callback;
    $self->{head_callback} = $head_callback;
    $self->{rec_callback} = $rec_callback;
    $query ||= {};
    my $hidden = $self->hidden();
    my ($key,$val);

    # this will overwrite any values for keys in {hidden} with those values in $q
    while (($key,$val) = each(%$query))
    {
        $hidden->{$key} = $val;
    }
    $self->{hidden} = $self->normalize_query($hidden);
    $self->{struct_query} = $self->structure_query($self->{hidden});
    $self->prepare_export();
    return \$self->{html};
}

##at should have a <NOSCRIPT> html tag to explain that the header help links will not be accessible

sub prepare_export
{
    my ($self) = @_;
    $self->reset_group();
    my ($group,$field_s);
    while ($group = $self->next_group())
    {
        my $record;
        my $rec_no;
        my $rec_count = $#{$group->{data}};
##at sharing $hidden with $record (in add_record()) is a bad idea could cause discrepancies below
##at are you sure?? I think if the user keeps their own namespace when using hidden there should be no problems???
##at what is really happening is that whatever is in hidden will take precedence over parameter keys in $record
        while ($record = $self->next_record())
        {
            my $head_rec;
            my $fields;
            my $tail_rec = 0;
            if ($group->{tabular} && $rec_no == 0)
            {
                (($group->{body_html} = "No records found"),next) unless %$record;
                $head_rec = '<THEAD><TR>';
            }
            if ($group->{tabular} && $rec_no == $rec_count)
            {
                $tail_rec = 1;
            }
            while ($field_s = $self->next_field())
            {
                $fields .= $self->field_group_html($field_s,$group,undef,($head_rec ? \$head_rec : undef()),$tail_rec);
            }
            if ($group->{tabular})
            {
                if ($rec_no == 0)
                {
                    $head_rec .= '</TR></THEAD>';
                    $group->{body_html} .= $head_rec;
                }
                $fields = &{$self->{rec_callback}}($fields,$record,$group,$self) if ref($self->{rec_callback}) eq 'CODE';
                $fields = "<TR" . ($tail_rec ? ' '.$group->{TABULAR_TR_TAIL_ATTRS} . ' CLASS="GTAIL_TR"' : '') . ">$fields</TR>";
            }
            $group->{body_html} .= $fields;
            $rec_no++;
        }
        $group->{body_html} = qq[<SCRIPT LANGUAGE="JavaScript"><!--\r\n$group->{js}\r\n--></SCRIPT>$group->{body_html}] if $group->{js};
##at for each css class there should be a form/group object attribute
##at with the same name as the class, any HTML attributes it contains will be inserted into the tag after the CLASS attr
        # GT => group table GH => Group Heading
        $group->{head_html} = qq[<DIV CLASS="GH">$group->{heading}</DIV><TABLE CLASS="GT" $self->{GT}>] unless defined($group->{head_html});
        $group->{tail_html} = qq[</TABLE>] unless defined($group->{tail_html});
        $group->{html} = "$group->{head_html}$group->{body_html}$group->{tail_html}";
        $self->{body_html} .= "<P>$group->{html}</P>";
    }
    $self->{heading} = qq[<H2>$self->{heading}</H2>] if $self->{heading};
    $self->{head_html} = qq[$self->{heading}<FORM NAME="$self->{name}" ACTION="$self->{action}" METHOD="POST" $self->{form_enctype_attr}>]
        unless defined($self->{head_html});
    if ($self->{valid_error})
    {
        $self->{verr_msg} = '!----INPUT ERROR: Please correct the fields marked below----!' unless $self->{verr_msg};
        $self->{head_html} .= qq[<P><B><FONT COLOR="RED"><DIV ALIGN="CENTER" CLASS="VERR_MSG">$self->{verr_msg}</DIV></FONT></B></P>];
    }
    my $submit_val = $self->escape($self->{submit_value});
    $submit_val = qq[VALUE="$submit_val"] if length($self->{submit_value});
##at shouldn't use WIDTH and ALIGN attrs - use classes/CSS instead
    $self->{tail_html} = qq[<P><TABLE WIDTH="100%"><TR><TD style="text-align: right;"><INPUT $self->{submit_button_attrs} TYPE="RESET"></TD><TD style="width: 30px;"></TD>] .
        qq[<TD style="text-align: left;"><INPUT $self->{submit_button_attrs} TYPE="SUBMIT" $submit_val></TD></TR></TABLE></P></FORM>] unless defined($self->{tail_html});
    $self->{tail_html} .= qq[<DIV>* Indicates required field</DIV>] if $self->{ast_foot};
    my $wmess = $self->escape("** Field accepts '\%' as a wildcard matching operator");
    $self->{tail_html} .= qq[<DIV>$wmess</DIV>] if $self->{_ast_match};
    $self->{body_html} = $self->hidden_html() . $self->{body_html};
    my $tmess = $self->{top_message};
    my $noscript = '';
    $noscript = qq[<NOSCRIPT><P>$self->{noscript}</NOSCRIPT>] if $self->{noscript};
    $self->{html} = "$noscript$tmess$self->{head_html}$self->{heading2}$self->{body_html}$self->{tail_html}";
    $self->reset_group();
}

sub field_group_html
{
    my ($self,$field_s,$group,$label_class_add,$head_rec,$tail_rec) = @_;
    my ($val,$head,$headadd);
    if (ref($field_s) eq 'ARRAY')
    {
        if (ref($field_s->[1]) eq 'ARRAY')
        {
            if ($self->group_usage() eq $CGI::AutoForm::SEARCH_GROUP)
            {
                # SEARCHABLE date range group
                ($headadd,$val) = $self->search_date_html($field_s,$group);
                $field_s = $field_s->[0];
            }
            elsif ($field_s->[1][0]{FIELD_GROUP} eq 'CONFIRM')
            {
                # For a confirmable date group (will this EVER happen!!)
                my $ret = $self->field_group_html($field_s->[0],$group);

                # using CSS content generation, can set a different heading for CONFIRM fields in the style sheet by setting a class property
                $ret .= $self->field_group_html($field_s->[1],$group,'CONFIRM') unless $group->{tabular};
                return $ret;
            }
            else { die "A horrible death"; }
        }
        elsif (substr($field_s->[0]{FORM_ELEMENT_NAME},-6) eq '_MONTH')
        {
            # Must be a date group
            $val .= '<TABLE BORDER="0" CELLSPACING="0" CELLPADDING="0"><TR>';
            $val .= $self->date_group_html($field_s);
            $val .= '</TR></TABLE>';
            $field_s = $field_s->[0];
        }
        elsif ($self->group_usage() eq $CGI::AutoForm::SEARCH_GROUP)
        {
            # Must be a plain search range with 'TEXT' controls
            $val = $self->search_range_html($field_s,$group);
            $field_s = $field_s->[0];
        }
        elsif ($field_s->[0]{FIELD_GROUP} eq 'CONFIRM')
        {
            my $ret = $self->field_group_html($field_s->[0],$group);
            $ret .= $self->field_group_html($field_s->[1],$group,'CONFIRM') unless $group->{tabular};
            return $ret;
        }
        else { die "A horrible death"; }
    }
    else
    {
        $val = $self->field_html($field_s);
    }
    
    if ($group->{tabular})
    {
        return '' unless length($field_s->{BRIEF_HEADING});
        if ($head_rec)
        {
            $head = $self->escape($field_s->{BRIEF_HEADING});
##at could also support TABULAR_TH_ATTRS, TABULAR_TD_ATTRS, etc management by adding it to UI_TABLE_COLUMN and pod up
            $$head_rec .= qq[<TH CLASS="TABULAR_TH" $group->{TABULAR_TH_ATTRS} $field_s->{TABULAR_TH_ATTRS}>$head</TH>];
        }
        $val = &{$self->{val_callback}}($val,$field_s,$self) if ref($self->{val_callback});
        if ($tail_rec && $group->{TABULAR_TD_TAIL_STYLE})
        {
            $field_s->{TABULAR_TD_STYLE} .= $group->{TABULAR_TD_TAIL_STYLE};
        }
        my $style = '';
        if ($field_s->{TABULAR_TD_STYLE})
        {
            $style = qq[style="$field_s->{TABULAR_TD_STYLE}"];
        }
        return "<TD" . ($field_s->{VALID_ERROR} ? ' CLASS="VERR TABULAR_TD"' : ' CLASS="TABULAR_TD"') . " $field_s->{TABULAR_TD_ATTRS}" .
            ($tail_rec ? ' '. $group->{TABULAR_TD_TAIL_ATTRS} : '') . " $style>" . (length($val) ? $val : '&nbsp;' ) . "</TD>";
    }
    else
    {
        my $valerr = $self->escape($field_s->{VALID_ERROR});
        $head = $self->escape($field_s->{HEADING});
        $head = $self->_process_field_head($field_s,$head,$group);

        # all callbacks must be responsible for escaping any added HTML
        $val = &{$self->{val_callback}}($val,$field_s,$self) if ref($self->{val_callback});

        my $class;
        $class .= "$label_class_add " if $label_class_add;
        $class .= "REQ " if $field_s->{REQUIRED} eq 'Y';
        $class .= "VERR " if $valerr;
        chop($class);
        $class = qq[ CLASS="$class" ] if $class;
        $headadd = qq[<P>$headadd</P>] if $headadd;

        # this can be overridden on-the-fly also
        $valerr = qq[<FONT COLOR="RED" CLASS="VERR"><SPAN CLASS="VERR">$valerr</SPAN></FONT>]
            if $valerr;

        my $ret = qq[<TD CLASS="VFL" $self->{VFL}><LABEL$class>$head $valerr</LABEL>$headadd</TD><TD CLASS="VFE" $self->{VFE}>&nbsp;$val</TD>];
        $ret = &{$self->{rec_callback}}($ret,$field_s,$group,$self) if ref($self->{rec_callback}) eq 'CODE';
        return qq[<TR CLASS="VFR" $self->{VFR}>$ret</TR>];
    }
}

# If callback differs, don't add alert_summary
sub _process_field_head
{
    my ($self,$field_s,$head,$group) = @_;
    # {head_callback} does not apply to tabular headings
    if (ref($self->{head_callback}))
    {
        my $call_head = &{$self->{head_callback}}($head,$field_s,$self);
        return $call_head if $head ne $call_head;
    }
    else
    {
        $head = qq[$head:&nbsp;];
        my $usage = $self->group_usage();
        if (!$self->readonly() && $field_s->{REQUIRED} eq 'Y' && ($usage eq $CGI::AutoForm::INSERT_GROUP || $usage eq $CGI::AutoForm::EDIT_GROUP))
        {
            $head .= '*';
            $self->{ast_foot}++;
        }
    }
    if (length($field_s->{HELP_SUMMARY}))
    {
        my $sum = $field_s->{HELP_SUMMARY};
        $sum =~ s/\"/\\"/gs;
        $sum =~ s/\r?\n/\\n/gs;
        my $func_name = $field_s->{FORM_ELEMENT_NAME};
        $func_name =~ s/\./_/gs;
        $func_name = "${func_name}_ALRT_SUM";
        $group->{js} .= qq[function $func_name(){alert("$sum");}\n];
        $head = qq[<A CLASS="AS" HREF="javascript:$func_name();">$head </A>];
    }
    return $head;
}

sub require_js
{
    my ($self) = @_;
    $self->{noscript} = qq[This application requires that you enable javascript in your browser.<BR>Refer to your browser's documentation] .
        qq[ to enable javascript and then return to this page.];
}

sub search_date_html
{
    my ($self,$field_s,$group) = @_;
    my ($val,$head);
    $val .= '<TABLE BORDER="0" CELLSPACING="0" CELLPADDING="0">';
    $val .= '<TR><TD>Between&nbsp;</TD>' . $self->date_group_html($field_s->[1]) . '</TR>';
    $val .= '<TR><TD>And</TD>' . $self->date_group_html($field_s->[2]) . '</TR>';
    $val .= '<TR><TD>OR within the past&nbsp;</TD>' . $self->date_rel_html($field_s->[3]) . '</TR>';
    $val .= '</TABLE>';
    $head = $self->field_html($field_s->[0]);
    return ($head,$val);
}

sub search_range_html
{
    my ($self,$field_s,$group) = @_;
    my $val;
    $val .= '<TABLE BORDER="0" CELLSPACING="0" CELLPADDING="0">';
    $val .= '<TR><TD>Between&nbsp;</TD><TD>' . $self->field_html($field_s->[0]) . '</TD></TR>';
    $val .= '<TR><TD>And</TD><TD>' . $self->field_html($field_s->[1]) . '</TD></TR>';
    $val .= '</TABLE>';
    return $val;
}

##at should include javascript functions for days of month, leap year, etc
sub date_group_html
{
    my ($self,$dateset) = @_;
    my $ret;
    foreach my $date (@$dateset)
    {
        my ($tit) = $date->{FORM_ELEMENT_NAME} =~ /\.(\w+)$/;
        my $fill;
        if ($tit eq '_MONTH')
        {
            $tit = 'Month';
            $fill = '-';
        }
        elsif ($tit eq '_DAY')
        {
            $tit = 'Day';
            $fill = '-';
        }
        elsif ($tit eq '_YEAR')
        {
            $tit = 'Year';
            $fill = '&nbsp;';
        }
        elsif ($tit eq '_HOUR')
        {
            $tit = 'Hour';
            $fill = ':';
        }
        elsif ($tit eq '_MIN')
        {
            $tit = 'Min';
            $fill = '';
        }
        $ret .= "<TD><DIV>$tit</DIV>". $self->field_html($date) . "$fill</TD>";
    }
    return $ret;
}

sub date_rel_html
{
    my ($self,$dateset) = @_;
    my $ret;
    foreach my $date (@$dateset)
    {
        my ($tit) = $date->{FORM_ELEMENT_NAME} =~ /\.(\w+)$/;
        my $fill;
        if ($tit eq '_QUANT')
        {
            $tit = 'Quantity';
            $fill = '';
            $ret .= "<TD><DIV>$tit</DIV>". $self->field_html($date) . "$fill</TD>";
        }
        elsif ($tit eq '_UNIT')
        {
            $tit = 'Units';
            $fill = '';
            $ret .= "<TD COLSPAN=\"2\"><DIV>$tit</DIV>". $self->field_html($date) . "$fill</TD>";
        }
    }
    return $ret;
}