[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[PATCH 3/3] lexer: Change the functions for retrieving token strings.
|
From: |
Ben Pfaff |
|
Subject: |
[PATCH 3/3] lexer: Change the functions for retrieving token strings. |
|
Date: |
Fri, 19 Nov 2010 21:06:08 -0800 |
Until now, lex_tokid() has been for T_ID tokens only and lex_tokstr() has
been for T_ID and T_STRING tokens. For T_ID tokens, lex_tokid() and
lex_tokstr() had slightly different semantics.
This doesn't entirely make sense, and these particular functions are not
the ones wanted most by clients, so this commit removes these functions
in favor of lex_tokcstr() and lex_tokss(), which are both applicable
to both T_ID and T_STRING tokens, with the same semantics in each case.
These functions are also easier for the upcoming reimplementation of the
lexer.
---
src/language/command.c | 6 ++--
src/language/control/loop.c | 4 +-
src/language/control/repeat.c | 14 +++----
src/language/data-io/combine-files.c | 6 ++--
src/language/data-io/data-list.c | 10 +++---
src/language/data-io/file-handle.q | 13 ++++---
src/language/data-io/get-data.c | 21 ++++++------
src/language/data-io/print.c | 2 +-
src/language/data-io/save-translate.c | 8 ++--
src/language/data-io/trim.c | 6 ++--
src/language/dictionary/attributes.c | 8 ++--
src/language/dictionary/missing-values.c | 4 +-
src/language/dictionary/modify-variables.c | 2 +-
src/language/dictionary/mrsets.c | 16 ++++----
src/language/dictionary/value-labels.c | 2 +-
src/language/dictionary/variable-label.c | 2 +-
src/language/dictionary/vector.c | 10 +++---
src/language/expressions/evaluate.c | 4 +-
src/language/expressions/parse.c | 22 ++++++------
src/language/lexer/format-parser.c | 6 ++--
src/language/lexer/lexer.c | 49 +++++++++++++--------------
src/language/lexer/lexer.h | 4 +-
src/language/lexer/q2c.c | 14 ++++----
src/language/lexer/value-parser.c | 9 ++++-
src/language/lexer/variable-parser.c | 14 ++++----
src/language/stats/aggregate.c | 6 ++--
src/language/stats/crosstabs.q | 2 +-
src/language/stats/descriptives.c | 6 ++--
src/language/stats/examine.q | 6 ++--
src/language/stats/frequencies.q | 7 ++--
src/language/stats/npar.c | 16 ++++----
src/language/stats/rank.q | 10 +++---
src/language/stats/regression.q | 2 +-
src/language/stats/reliability.c | 2 +-
src/language/stats/sort-criteria.c | 2 +-
src/language/tests/float-format.c | 6 ++--
src/language/tests/format-guesser-test.c | 4 +-
src/language/tests/paper-size.c | 4 +-
src/language/utilities/cd.c | 4 +-
src/language/utilities/echo.c | 2 +-
src/language/utilities/host.c | 2 +-
src/language/utilities/include.c | 4 +-
src/language/utilities/permissions.c | 2 +-
src/language/utilities/set.q | 15 ++++----
src/language/utilities/title.c | 4 +-
src/language/xforms/compute.c | 6 ++--
src/language/xforms/count.c | 6 ++--
src/language/xforms/recode.c | 22 ++++++------
48 files changed, 200 insertions(+), 196 deletions(-)
diff --git a/src/language/command.c b/src/language/command.c
index 3da5da6..3bf9571 100644
--- a/src/language/command.c
+++ b/src/language/command.c
@@ -288,7 +288,7 @@ parse_command_name (struct lexer *lexer)
{
if (!ds_is_empty (&s) && ds_last (&s) != '-')
ds_put_byte (&s, ' ');
- ds_put_cstr (&s, lex_tokid (lexer));
+ ds_put_cstr (&s, lex_tokcstr (lexer));
}
else if (lex_is_integer (lexer) && lex_integer (lexer) >= 0)
{
@@ -514,10 +514,10 @@ cmd_erase (struct lexer *lexer, struct dataset *ds UNUSED)
if (!lex_force_string (lexer))
return CMD_FAILURE;
- if (remove (ds_cstr (lex_tokstr (lexer))) == -1)
+ if (remove (lex_tokcstr (lexer)) == -1)
{
msg (SW, _("Error removing `%s': %s."),
- ds_cstr (lex_tokstr (lexer)), strerror (errno));
+ lex_tokcstr (lexer), strerror (errno));
return CMD_FAILURE;
}
diff --git a/src/language/control/loop.c b/src/language/control/loop.c
index f17542c..5e99232 100644
--- a/src/language/control/loop.c
+++ b/src/language/control/loop.c
@@ -221,13 +221,13 @@ parse_index_clause (struct dataset *ds, struct lexer
*lexer,
return false;
}
- loop->index_var = dict_lookup_var (dataset_dict (ds), lex_tokid (lexer));
+ loop->index_var = dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer));
if (loop->index_var != NULL)
*created_index_var = false;
else
{
loop->index_var = dict_create_var_assert (dataset_dict (ds),
- lex_tokid (lexer), 0);
+ lex_tokcstr (lexer), 0);
*created_index_var = true;
}
lex_get (lexer);
diff --git a/src/language/control/repeat.c b/src/language/control/repeat.c
index ddbba09..130dfd6 100644
--- a/src/language/control/repeat.c
+++ b/src/language/control/repeat.c
@@ -168,21 +168,19 @@ parse_specification (struct lexer *lexer, struct
repeat_block *block)
/* Get a stand-in variable name and make sure it's unique. */
if (!lex_force_id (lexer))
return false;
- if (dict_lookup_var (dict, lex_tokid (lexer)))
- msg (SW, _("Dummy variable name `%s' hides dictionary "
- "variable `%s'."),
- lex_tokid (lexer), lex_tokid (lexer));
- if (find_macro (block, ss_cstr (lex_tokid (lexer))))
+ if (dict_lookup_var (dict, lex_tokcstr (lexer)))
+ msg (SW, _("Dummy variable name `%s' hides dictionary variable `%s'."),
+ lex_tokcstr (lexer), lex_tokcstr (lexer));
+ if (find_macro (block, lex_tokss (lexer)))
{
msg (SE, _("Dummy variable name `%s' is given twice."),
- lex_tokid (lexer));
+ lex_tokcstr (lexer));
return false;
}
/* Make a new macro. */
macro = pool_alloc (block->pool, sizeof *macro);
- ss_alloc_substring_pool (¯o->name, ss_cstr (lex_tokid (lexer)),
- block->pool);
+ ss_alloc_substring_pool (¯o->name, lex_tokss (lexer), block->pool);
ll_push_tail (&block->macros, ¯o->ll);
/* Skip equals sign. */
diff --git a/src/language/data-io/combine-files.c
b/src/language/data-io/combine-files.c
index 66163ee..aa6d3b9 100644
--- a/src/language/data-io/combine-files.c
+++ b/src/language/data-io/combine-files.c
@@ -251,7 +251,7 @@ combine_files (enum comb_command_type command,
"TABLE."));
goto error;
}
- strcpy (file->in_name, lex_tokid (lexer));
+ strcpy (file->in_name, lex_tokcstr (lexer));
lex_get (lexer);
}
else if (lex_match_id (lexer, "SORT"))
@@ -325,7 +325,7 @@ combine_files (enum comb_command_type command,
lex_match (lexer, T_EQUALS);
if (!lex_force_id (lexer))
goto error;
- strcpy (first_name, lex_tokid (lexer));
+ strcpy (first_name, lex_tokcstr (lexer));
lex_get (lexer);
}
else if (command != COMB_UPDATE && lex_match_id (lexer, "LAST"))
@@ -339,7 +339,7 @@ combine_files (enum comb_command_type command,
lex_match (lexer, T_EQUALS);
if (!lex_force_id (lexer))
goto error;
- strcpy (last_name, lex_tokid (lexer));
+ strcpy (last_name, lex_tokcstr (lexer));
lex_get (lexer);
}
else if (lex_match_id (lexer, "MAP"))
diff --git a/src/language/data-io/data-list.c b/src/language/data-io/data-list.c
index aec243a..046c7b0 100644
--- a/src/language/data-io/data-list.c
+++ b/src/language/data-io/data-list.c
@@ -108,7 +108,7 @@ cmd_data_list (struct lexer *lexer, struct dataset *ds)
if (!lex_force_string (lexer))
goto error;
- ds_init_string (&encoding, lex_tokstr (lexer));
+ ds_init_substring (&encoding, lex_tokss (lexer));
lex_get (lexer);
}
@@ -147,9 +147,9 @@ cmd_data_list (struct lexer *lexer, struct dataset *ds)
lex_match (lexer, T_EQUALS);
if (!lex_force_id (lexer))
goto error;
- end = dict_lookup_var (dict, lex_tokid (lexer));
+ end = dict_lookup_var (dict, lex_tokcstr (lexer));
if (!end)
- end = dict_create_var_assert (dict, lex_tokid (lexer), 0);
+ end = dict_create_var_assert (dict, lex_tokcstr (lexer), 0);
lex_get (lexer);
}
else if (lex_match_id (lexer, "NOTABLE"))
@@ -197,9 +197,9 @@ cmd_data_list (struct lexer *lexer, struct dataset *ds)
if (lex_match_id (lexer, "TAB"))
delim = '\t';
else if (lex_is_string (lexer)
- && ds_length (lex_tokstr (lexer)) == 1)
+ && ss_length (lex_tokss (lexer)) == 1)
{
- delim = ds_first (lex_tokstr (lexer));
+ delim = ss_first (lex_tokss (lexer));
lex_get (lexer);
}
else
diff --git a/src/language/data-io/file-handle.q
b/src/language/data-io/file-handle.q
index ba84a15..5f2a440 100644
--- a/src/language/data-io/file-handle.q
+++ b/src/language/data-io/file-handle.q
@@ -57,7 +57,7 @@ cmd_file_handle (struct lexer *lexer, struct dataset *ds)
if (!lex_force_id (lexer))
return CMD_CASCADING_FAILURE;
- str_copy_trunc (handle_name, sizeof handle_name, lex_tokid (lexer));
+ str_copy_trunc (handle_name, sizeof handle_name, lex_tokcstr (lexer));
handle = fh_from_id (handle_name);
if (handle != NULL)
@@ -159,7 +159,7 @@ cmd_close_file_handle (struct lexer *lexer, struct dataset
*ds UNUSED)
if (!lex_force_id (lexer))
return CMD_CASCADING_FAILURE;
- handle = fh_from_id (lex_tokid (lexer));
+ handle = fh_from_id (lex_tokcstr (lexer));
if (handle == NULL)
return CMD_CASCADING_FAILURE;
@@ -208,14 +208,15 @@ fh_parse (struct lexer *lexer, enum fh_referent
referent_mask)
handle = NULL;
if (lex_token (lexer) == T_ID)
- handle = fh_from_id (lex_tokid (lexer));
+ handle = fh_from_id (lex_tokcstr (lexer));
if (handle == NULL)
{
- if (lex_token (lexer) != T_ID || lex_tokid (lexer)[0] != '#' ||
settings_get_syntax () != ENHANCED)
- handle = fh_create_file (NULL, ds_cstr (lex_tokstr (lexer)),
+ if (lex_token (lexer) != T_ID || lex_tokcstr (lexer)[0] != '#'
+ || settings_get_syntax () != ENHANCED)
+ handle = fh_create_file (NULL, lex_tokcstr (lexer),
fh_default_properties ());
else
- handle = fh_create_scratch (lex_tokid (lexer));
+ handle = fh_create_scratch (lex_tokcstr (lexer));
}
lex_get (lexer);
}
diff --git a/src/language/data-io/get-data.c b/src/language/data-io/get-data.c
index eeb3827..05e115a 100644
--- a/src/language/data-io/get-data.c
+++ b/src/language/data-io/get-data.c
@@ -61,7 +61,7 @@ cmd_get_data (struct lexer *lexer, struct dataset *ds)
else if (lex_match_id (lexer, "PSQL"))
return parse_get_psql (lexer, ds);
- msg (SE, _("Unsupported TYPE %s"), lex_tokid (lexer));
+ msg (SE, _("Unsupported TYPE %s"), lex_tokcstr (lexer));
return CMD_FAILURE;
}
@@ -85,7 +85,7 @@ parse_get_psql (struct lexer *lexer, struct dataset *ds)
if (!lex_force_string (lexer))
goto error;
- psql.conninfo = xstrdup (ds_cstr (lex_tokstr (lexer)));
+ psql.conninfo = ss_xstrdup (lex_tokss (lexer));
lex_get (lexer);
@@ -113,7 +113,7 @@ parse_get_psql (struct lexer *lexer, struct dataset *ds)
if ( ! lex_force_string (lexer) )
goto error;
- ds_put_substring (&psql.sql, lex_tokstr (lexer)->ss);
+ ds_put_substring (&psql.sql, lex_tokss (lexer));
lex_get (lexer);
}
}
@@ -153,7 +153,7 @@ parse_get_gnm (struct lexer *lexer, struct dataset *ds)
if (!lex_force_string (lexer))
goto error;
- gri.file_name = xstrdup (ds_cstr (lex_tokstr (lexer)));
+ gri.file_name = ss_xstrdup (lex_tokss (lexer));
lex_get (lexer);
@@ -172,7 +172,7 @@ parse_get_gnm (struct lexer *lexer, struct dataset *ds)
if ( ! lex_force_string (lexer) )
goto error;
- gri.sheet_name = xstrdup (ds_cstr (lex_tokstr (lexer)));
+ gri.sheet_name = ss_xstrdup (lex_tokss (lexer));
gri.sheet_index = -1;
}
else if (lex_match_id (lexer, "INDEX"))
@@ -196,7 +196,7 @@ parse_get_gnm (struct lexer *lexer, struct dataset *ds)
if ( ! lex_force_string (lexer) )
goto error;
- gri.cell_range = xstrdup (ds_cstr (lex_tokstr (lexer)));
+ gri.cell_range = ss_xstrdup (lex_tokss (lexer));
}
else
goto error;
@@ -219,6 +219,7 @@ parse_get_gnm (struct lexer *lexer, struct dataset *ds)
}
else
{
+ printf ("Unknown data file type `%s'\n", lex_tokcstr (lexer));
goto error;
}
lex_get (lexer);
@@ -415,7 +416,7 @@ parse_get_txt (struct lexer *lexer, struct dataset *ds)
if (!lex_force_string (lexer))
goto error;
- s = ds_ss (lex_tokstr (lexer));
+ s = lex_tokss (lexer);
if (ss_match_string (&s, ss_cstr ("\\t")))
ds_put_cstr (&hard_seps, "\t");
if (ss_match_string (&s, ss_cstr ("\\\\")))
@@ -441,14 +442,14 @@ parse_get_txt (struct lexer *lexer, struct dataset *ds)
goto error;
if (settings_get_syntax () == COMPATIBLE
- && ds_length (lex_tokstr (lexer)) != 1)
+ && ss_length (lex_tokss (lexer)) != 1)
{
msg (SE, _("In compatible syntax mode, the QUALIFIER string "
"must contain exactly one character."));
goto error;
}
- data_parser_set_quotes (parser, ds_ss (lex_tokstr (lexer)));
+ data_parser_set_quotes (parser, lex_tokss (lexer));
lex_get (lexer);
}
else if (settings_get_syntax () == ENHANCED
@@ -501,7 +502,7 @@ parse_get_txt (struct lexer *lexer, struct dataset *ds)
if (!lex_force_id (lexer))
goto error;
- strcpy (name, lex_tokid (lexer));
+ strcpy (name, lex_tokcstr (lexer));
lex_get (lexer);
if (type == DP_DELIMITED)
diff --git a/src/language/data-io/print.c b/src/language/data-io/print.c
index cfbe043..96ac15e 100644
--- a/src/language/data-io/print.c
+++ b/src/language/data-io/print.c
@@ -280,7 +280,7 @@ parse_string_argument (struct lexer *lexer, struct
print_trns *trns, int record,
spec->type = PRT_LITERAL;
spec->record = record;
spec->first_column = *column;
- ds_init_string (&spec->string, lex_tokstr (lexer));
+ ds_init_substring (&spec->string, lex_tokss (lexer));
ds_register_pool (&spec->string, trns->pool);
lex_get (lexer);
diff --git a/src/language/data-io/save-translate.c
b/src/language/data-io/save-translate.c
index dcfdb83..213f0bf 100644
--- a/src/language/data-io/save-translate.c
+++ b/src/language/data-io/save-translate.c
@@ -159,13 +159,13 @@ cmd_save_translate (struct lexer *lexer, struct dataset
*ds)
lex_match (lexer, T_EQUALS);
if (!lex_force_string (lexer))
goto error;
- if (ds_length (lex_tokstr (lexer)) != 1)
+ if (ss_length (lex_tokss (lexer)) != 1)
{
msg (SE, _("The %s string must contain exactly one "
"character."), "DELIMITER");
goto error;
}
- delimiter = ds_first (lex_tokstr (lexer));
+ delimiter = ss_first (lex_tokss (lexer));
lex_get (lexer);
}
else if (lex_match_id (lexer, "QUALIFIER"))
@@ -173,13 +173,13 @@ cmd_save_translate (struct lexer *lexer, struct dataset
*ds)
lex_match (lexer, T_EQUALS);
if (!lex_force_string (lexer))
goto error;
- if (ds_length (lex_tokstr (lexer)) != 1)
+ if (ss_length (lex_tokss (lexer)) != 1)
{
msg (SE, _("The %s string must contain exactly one "
"character."), "QUALIFIER");
goto error;
}
- qualifier = ds_first (lex_tokstr (lexer));
+ qualifier = ss_first (lex_tokss (lexer));
lex_get (lexer);
}
else if (lex_match_id (lexer, "DECIMAL"))
diff --git a/src/language/data-io/trim.c b/src/language/data-io/trim.c
index be637c4..9f76b10 100644
--- a/src/language/data-io/trim.c
+++ b/src/language/data-io/trim.c
@@ -83,18 +83,18 @@ parse_dict_rename (struct lexer *lexer, struct dictionary
*dict)
if (!lex_force_match (lexer, T_EQUALS)
|| !lex_force_id (lexer))
return 0;
- if (dict_lookup_var (dict, lex_tokid (lexer)) != NULL)
+ if (dict_lookup_var (dict, lex_tokcstr (lexer)) != NULL)
{
msg (SE, _("Cannot rename %s as %s because there already exists "
"a variable named %s. To rename variables with "
"overlapping names, use a single RENAME subcommand "
"such as `/RENAME (A=B)(B=C)(C=A)', or equivalently, "
"`/RENAME (A B C=B C A)'."),
- var_get_name (v), lex_tokid (lexer), lex_tokid (lexer));
+ var_get_name (v), lex_tokcstr (lexer), lex_tokcstr (lexer));
return 0;
}
- dict_rename_var (dict, v, lex_tokid (lexer));
+ dict_rename_var (dict, v, lex_tokcstr (lexer));
lex_get (lexer);
return 1;
}
diff --git a/src/language/dictionary/attributes.c
b/src/language/dictionary/attributes.c
index 35e6954..c992ff8 100644
--- a/src/language/dictionary/attributes.c
+++ b/src/language/dictionary/attributes.c
@@ -79,7 +79,7 @@ static bool
match_subcommand (struct lexer *lexer, const char *keyword)
{
if (lex_token (lexer) == T_ID
- && lex_id_match (ss_cstr (lex_tokid (lexer)), ss_cstr (keyword))
+ && lex_id_match (lex_tokss (lexer), ss_cstr (keyword))
&& lex_look_ahead (lexer) == T_EQUALS)
{
lex_get (lexer); /* Skip keyword. */
@@ -96,7 +96,7 @@ parse_attribute_name (struct lexer *lexer, char
name[VAR_NAME_LEN + 1],
{
if (!lex_force_id (lexer))
return false;
- strcpy (name, lex_tokid (lexer));
+ strcpy (name, lex_tokcstr (lexer));
lex_get (lexer);
if (lex_match (lexer, T_LBRACK))
@@ -122,14 +122,14 @@ static bool
add_attribute (struct lexer *lexer, struct attrset **sets, size_t n)
{
char name[VAR_NAME_LEN + 1];
+ const char *value;
size_t index, i;
- char *value;
if (!parse_attribute_name (lexer, name, &index)
|| !lex_force_match (lexer, T_LPAREN)
|| !lex_force_string (lexer))
return false;
- value = ds_cstr (lex_tokstr (lexer));
+ value = lex_tokcstr (lexer);
for (i = 0; i < n; i++)
{
diff --git a/src/language/dictionary/missing-values.c
b/src/language/dictionary/missing-values.c
index bbc2c84..4a14ee1 100644
--- a/src/language/dictionary/missing-values.c
+++ b/src/language/dictionary/missing-values.c
@@ -106,7 +106,7 @@ cmd_missing_values (struct lexer *lexer, struct dataset *ds)
break;
}
- length = ds_length (lex_tokstr (lexer));
+ length = ss_length (lex_tokss (lexer));
if (length > MV_MAX_STRING)
{
msg (SE, _("Truncating missing value to maximum "
@@ -115,7 +115,7 @@ cmd_missing_values (struct lexer *lexer, struct dataset *ds)
length = MV_MAX_STRING;
}
memset (value, ' ', MV_MAX_STRING);
- memcpy (value, ds_data (lex_tokstr (lexer)), length);
+ memcpy (value, ss_data (lex_tokss (lexer)), length);
if (!mv_add_str (&mv, value))
deferred_errors = true;
diff --git a/src/language/dictionary/modify-variables.c
b/src/language/dictionary/modify-variables.c
index 6afd321..cc84737 100644
--- a/src/language/dictionary/modify-variables.c
+++ b/src/language/dictionary/modify-variables.c
@@ -300,7 +300,7 @@ cmd_modify_vars (struct lexer *lexer, struct dataset *ds)
else
{
if (lex_token (lexer) == T_ID)
- msg (SE, _("Unrecognized subcommand name `%s'."), lex_tokid
(lexer));
+ msg (SE, _("Unrecognized subcommand name `%s'."), lex_tokcstr
(lexer));
else
msg (SE, _("Subcommand name expected."));
goto done;
diff --git a/src/language/dictionary/mrsets.c b/src/language/dictionary/mrsets.c
index 0708e89..c775f49 100644
--- a/src/language/dictionary/mrsets.c
+++ b/src/language/dictionary/mrsets.c
@@ -93,16 +93,16 @@ parse_group (struct lexer *lexer, struct dictionary *dict,
{
if (!lex_force_match (lexer, T_EQUALS) || !lex_force_id (lexer))
goto error;
- if (lex_tokid (lexer)[0] != '$')
+ if (lex_tokcstr (lexer)[0] != '$')
{
msg (SE, _("%s is not a valid name for a multiple response "
"set. Multiple response set names must begin with "
- "`$'."), lex_tokid (lexer));
+ "`$'."), lex_tokcstr (lexer));
goto error;
}
free (mrset->name);
- mrset->name = xstrdup (lex_tokid (lexer));
+ mrset->name = xstrdup (lex_tokcstr (lexer));
lex_get (lexer);
}
else if (lex_match_id (lexer, "VARIABLES"))
@@ -129,7 +129,7 @@ parse_group (struct lexer *lexer, struct dictionary *dict,
goto error;
free (mrset->label);
- mrset->label = ds_xstrdup (lex_tokstr (lexer));
+ mrset->label = ss_xstrdup (lex_tokss (lexer));
lex_get (lexer);
}
else if (type == MRSET_MD && lex_match_id (lexer, "LABELSOURCE"))
@@ -159,7 +159,7 @@ parse_group (struct lexer *lexer, struct dictionary *dict,
}
else if (lex_is_string (lexer))
{
- const char *s = ds_cstr (lex_tokstr (lexer));
+ const char *s = lex_tokcstr (lexer);
int width;
/* Trim off trailing spaces, but don't trim the string until
@@ -480,14 +480,14 @@ parse_mrset_names (struct lexer *lexer, struct dictionary
*dict,
{
if (!lex_force_id (lexer))
return false;
- if (dict_lookup_mrset (dict, lex_tokid (lexer)) == NULL)
+ if (dict_lookup_mrset (dict, lex_tokcstr (lexer)) == NULL)
{
msg (SE, _("No multiple response set named %s."),
- lex_tokid (lexer));
+ lex_tokcstr (lexer));
stringi_set_destroy (mrset_names);
return false;
}
- stringi_set_insert (mrset_names, lex_tokid (lexer));
+ stringi_set_insert (mrset_names, lex_tokcstr (lexer));
lex_get (lexer);
}
}
diff --git a/src/language/dictionary/value-labels.c
b/src/language/dictionary/value-labels.c
index 0fdd7a2..1471fc8 100644
--- a/src/language/dictionary/value-labels.c
+++ b/src/language/dictionary/value-labels.c
@@ -142,7 +142,7 @@ get_label (struct lexer *lexer, struct variable **vars,
size_t var_cnt)
return 0;
}
- ds_init_string (&label, lex_tokstr (lexer));
+ ds_init_substring (&label, lex_tokss (lexer));
if (ds_length (&label) > 60)
{
diff --git a/src/language/dictionary/variable-label.c
b/src/language/dictionary/variable-label.c
index e6bd622..804a421 100644
--- a/src/language/dictionary/variable-label.c
+++ b/src/language/dictionary/variable-label.c
@@ -52,7 +52,7 @@ cmd_variable_labels (struct lexer *lexer, struct dataset *ds)
return CMD_FAILURE;
}
- ds_init_string (&label, lex_tokstr (lexer) );
+ ds_init_substring (&label, lex_tokss (lexer));
if (ds_length (&label) > 255)
{
msg (SW, _("Truncating variable label to 255 characters."));
diff --git a/src/language/dictionary/vector.c b/src/language/dictionary/vector.c
index a884331..3be0723 100644
--- a/src/language/dictionary/vector.c
+++ b/src/language/dictionary/vector.c
@@ -59,25 +59,25 @@ cmd_vector (struct lexer *lexer, struct dataset *ds)
{
size_t i;
- if (dict_lookup_vector (dict, lex_tokid (lexer)))
+ if (dict_lookup_vector (dict, lex_tokcstr (lexer)))
{
msg (SE, _("A vector named %s already exists."),
- lex_tokid (lexer));
+ lex_tokcstr (lexer));
goto fail;
}
for (i = 0; i < vector_cnt; i++)
- if (!strcasecmp (vectors[i], lex_tokid (lexer)))
+ if (!strcasecmp (vectors[i], lex_tokcstr (lexer)))
{
msg (SE, _("Vector name %s is given twice."),
- lex_tokid (lexer));
+ lex_tokcstr (lexer));
goto fail;
}
if (vector_cnt == vector_cap)
vectors = pool_2nrealloc (pool,
vectors, &vector_cap, sizeof *vectors);
- vectors[vector_cnt++] = pool_strdup (pool, lex_tokid (lexer));
+ vectors[vector_cnt++] = pool_strdup (pool, lex_tokcstr (lexer));
lex_get (lexer);
lex_match (lexer, T_COMMA);
diff --git a/src/language/expressions/evaluate.c
b/src/language/expressions/evaluate.c
index 9bcfe31..79b5215 100644
--- a/src/language/expressions/evaluate.c
+++ b/src/language/expressions/evaluate.c
@@ -134,7 +134,7 @@ cmd_debug_evaluate (struct lexer *lexer, struct dataset
*dsother UNUSED)
if (!lex_force_id (lexer))
goto done;
- strcpy (name, lex_tokid (lexer));
+ strcpy (name, lex_tokcstr (lexer));
lex_get (lexer);
if (!lex_force_match (lexer, T_EQUALS))
@@ -143,7 +143,7 @@ cmd_debug_evaluate (struct lexer *lexer, struct dataset
*dsother UNUSED)
if (lex_is_number (lexer))
width = 0;
else if (lex_is_string (lexer))
- width = ds_length (lex_tokstr (lexer));
+ width = ss_length (lex_tokss (lexer));
else
{
lex_error (lexer, _("expecting number or string"));
diff --git a/src/language/expressions/parse.c b/src/language/expressions/parse.c
index 79576b4..ae968d0 100644
--- a/src/language/expressions/parse.c
+++ b/src/language/expressions/parse.c
@@ -824,7 +824,7 @@ parse_sysvar (struct lexer *lexer, struct expression *e)
return expr_allocate_number (e, settings_get_viewwidth ());
else
{
- msg (SE, _("Unknown system variable %s."), lex_tokid (lexer));
+ msg (SE, _("Unknown system variable %s."), lex_tokcstr (lexer));
return NULL;
}
}
@@ -841,17 +841,17 @@ parse_primary (struct lexer *lexer, struct expression *e)
/* An identifier followed by a left parenthesis may be
a vector element reference. If not, it's a function
call. */
- if (e->ds != NULL && dict_lookup_vector (dataset_dict (e->ds),
lex_tokid (lexer)) != NULL)
+ if (e->ds != NULL && dict_lookup_vector (dataset_dict (e->ds),
lex_tokcstr (lexer)) != NULL)
return parse_vector_element (lexer, e);
else
return parse_function (lexer, e);
}
- else if (lex_tokid (lexer)[0] == '$')
+ else if (lex_tokcstr (lexer)[0] == '$')
{
/* $ at the beginning indicates a system variable. */
return parse_sysvar (lexer, e);
}
- else if (e->ds != NULL && dict_lookup_var (dataset_dict (e->ds),
lex_tokid (lexer)))
+ else if (e->ds != NULL && dict_lookup_var (dataset_dict (e->ds),
lex_tokcstr (lexer)))
{
/* It looks like a user variable.
(It could be a format specifier, but we'll assume
@@ -872,7 +872,7 @@ parse_primary (struct lexer *lexer, struct expression *e)
return expr_allocate_format (e, &fmt);
/* All attempts failed. */
- msg (SE, _("Unknown identifier %s."), lex_tokid (lexer));
+ msg (SE, _("Unknown identifier %s."), lex_tokcstr (lexer));
return NULL;
}
break;
@@ -888,7 +888,7 @@ parse_primary (struct lexer *lexer, struct expression *e)
case T_STRING:
{
union any_node *node = expr_allocate_string_buffer (
- e, ds_cstr (lex_tokstr (lexer) ), ds_length (lex_tokstr (lexer) ));
+ e, lex_tokcstr (lexer), ss_length (lex_tokss (lexer)));
lex_get (lexer);
return node;
}
@@ -918,7 +918,7 @@ parse_vector_element (struct lexer *lexer, struct
expression *e)
/* Find vector, skip token.
The caller must already have verified that the current token
is the name of a vector. */
- vector = dict_lookup_vector (dataset_dict (e->ds), lex_tokid (lexer));
+ vector = dict_lookup_vector (dataset_dict (e->ds), lex_tokcstr (lexer));
assert (vector != NULL);
lex_get (lexer);
@@ -1209,11 +1209,11 @@ parse_function (struct lexer *lexer, struct expression
*e)
union any_node *n;
- ds_init_string (&func_name, lex_tokstr (lexer));
- min_valid = extract_min_valid (ds_cstr (lex_tokstr (lexer)));
- if (!lookup_function (ds_cstr (lex_tokstr (lexer)), &first, &last))
+ ds_init_substring (&func_name, lex_tokss (lexer));
+ min_valid = extract_min_valid (lex_tokcstr (lexer));
+ if (!lookup_function (lex_tokcstr (lexer), &first, &last))
{
- msg (SE, _("No function or vector named %s."), ds_cstr (lex_tokstr
(lexer)));
+ msg (SE, _("No function or vector named %s."), lex_tokcstr (lexer));
ds_destroy (&func_name);
return NULL;
}
diff --git a/src/language/lexer/format-parser.c
b/src/language/lexer/format-parser.c
index 1730b49..4b25020 100644
--- a/src/language/lexer/format-parser.c
+++ b/src/language/lexer/format-parser.c
@@ -46,7 +46,7 @@ parse_abstract_format_specifier__ (struct lexer *lexer,
goto error;
/* Extract pieces. */
- s = ds_ss (lex_tokstr (lexer));
+ s = ss_cstr (lex_tokcstr (lexer));
ss_get_bytes (&s, ss_span (s, ss_cstr (CC_LETTERS)), &type_ss);
ss_get_bytes (&s, ss_span (s, ss_cstr (CC_DIGITS)), &width_ss);
if (ss_match_byte (&s, '.'))
@@ -133,9 +133,9 @@ parse_format_specifier_name (struct lexer *lexer, enum
fmt_type *type)
lex_error (lexer, _("expecting format type"));
return false;
}
- if (!fmt_from_name (ds_cstr (lex_tokstr (lexer)), type))
+ if (!fmt_from_name (lex_tokcstr (lexer), type))
{
- msg (SE, _("Unknown format type `%s'."), ds_cstr (lex_tokstr (lexer)));
+ msg (SE, _("Unknown format type `%s'."), lex_tokcstr (lexer));
return false;
}
lex_get (lexer);
diff --git a/src/language/lexer/lexer.c b/src/language/lexer/lexer.c
index c08c428..4a899bd 100644
--- a/src/language/lexer/lexer.c
+++ b/src/language/lexer/lexer.c
@@ -49,11 +49,7 @@ struct lexer
int token; /* Current token. */
double tokval; /* T_POS_NUM, T_NEG_NUM: the token's value. */
- char tokid [VAR_NAME_LEN + 1]; /* T_ID: the identifier. */
-
- struct string tokstr; /* T_ID, T_STRING: token string value.
- For T_ID, this is not truncated as is
- tokid. */
+ struct string tokstr; /* T_ID, T_STRING: token string value. */
char *prog; /* Pointer to next token in line_buffer. */
bool dot; /* True only if this line ends with a terminal dot. */
@@ -138,7 +134,6 @@ restore_token (struct lexer *lexer)
assert (lexer->put_token != 0);
lexer->token = lexer->put_token;
ds_assign_string (&lexer->tokstr, &lexer->put_tokstr);
- str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
lexer->tokval = lexer->put_tokval;
lexer->put_token = 0;
}
@@ -405,8 +400,7 @@ lex_get (struct lexer *lexer)
}
}
-/* Parses an identifier at the current position into tokid and
- tokstr.
+/* Parses an identifier at the current position into tokstr.
Returns the correct token type. */
static int
parse_id (struct lexer *lexer)
@@ -420,7 +414,6 @@ parse_id (struct lexer *lexer)
lexer->prog += ss_length (id);
ds_assign_substring (&lexer->tokstr, id);
- str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
return lex_id_to_token (id);
}
@@ -571,7 +564,7 @@ bool
lex_match_id_n (struct lexer *lexer, const char *s, size_t n)
{
if (lexer->token == T_ID
- && lex_id_match_n (ss_cstr (s), ss_cstr (lexer->tokid), n))
+ && lex_id_match_n (ss_cstr (s), lex_tokss (lexer), n))
{
lex_get (lexer);
return true;
@@ -798,7 +791,6 @@ lex_put_back_id (struct lexer *lexer, const char *id)
save_token (lexer);
lexer->token = T_ID;
ds_assign_cstr (&lexer->tokstr, id);
- str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr));
}
/* Weird line processing functions. */
@@ -1082,36 +1074,37 @@ lex_token_representation (struct lexer *lexer)
case T_ID:
case T_POS_NUM:
case T_NEG_NUM:
- return ds_xstrdup (&lexer->tokstr);
+ return ss_xstrdup (lex_tokss (lexer));
case T_STRING:
{
+ struct substring ss;
int hexstring = 0;
char *sp, *dp;
- for (sp = ds_cstr (&lexer->tokstr); sp < ds_end (&lexer->tokstr); sp++)
+ ss = lex_tokss (lexer);
+ for (sp = ss_data (ss); sp < ss_end (ss); sp++)
if (!c_isprint ((unsigned char) *sp))
{
hexstring = 1;
break;
}
- token_rep = xmalloc (2 + ds_length (&lexer->tokstr) * 2 + 1 + 1);
+ token_rep = xmalloc (2 + ss_length (ss) * 2 + 1 + 1);
dp = token_rep;
if (hexstring)
*dp++ = 'X';
*dp++ = '\'';
- if (!hexstring)
- for (sp = ds_cstr (&lexer->tokstr); *sp; )
+ for (sp = ss_data (ss); sp < ss_end (ss); sp++)
+ if (!hexstring)
{
if (*sp == '\'')
*dp++ = '\'';
- *dp++ = (unsigned char) *sp++;
+ *dp++ = (unsigned char) *sp;
}
- else
- for (sp = ds_cstr (&lexer->tokstr); sp < ds_end (&lexer->tokstr);
sp++)
+ else
{
*dp++ = (((unsigned char) *sp) >> 4)["0123456789ABCDEF"];
*dp++ = (((unsigned char) *sp) & 15)["0123456789ABCDEF"];
@@ -1344,16 +1337,22 @@ lex_tokval (const struct lexer *lexer)
return lexer->tokval;
}
+/* Returns the null-terminated string value associated with LEXER's current
+ token. For a T_ID token, this is the identifier, and for a T_STRING token,
+ this is the string. For other tokens the value is undefined. */
const char *
-lex_tokid (const struct lexer *lexer)
+lex_tokcstr (const struct lexer *lexer)
{
- return lexer->tokid;
+ return ds_cstr (&lexer->tokstr);
}
-const struct string *
-lex_tokstr (const struct lexer *lexer)
+/* Returns the string value associated with LEXER's current token. For a T_ID
+ token, this is the identifier, and for a T_STRING token, this is the string.
+ For other tokens the value is undefined. */
+struct substring
+lex_tokss (const struct lexer *lexer)
{
- return &lexer->tokstr;
+ return ds_ss (&lexer->tokstr);
}
/* If the lexer is positioned at the (pseudo)identifier S, which
@@ -1368,7 +1367,7 @@ lex_match_hyphenated_word (struct lexer *lexer, const
char *s)
if (hyphen == NULL)
return lex_match_id (lexer, s);
else if (lexer->token != T_ID
- || !lex_id_match (ss_buffer (s, hyphen - s), ss_cstr (lexer->tokid))
+ || !lex_id_match (ss_buffer (s, hyphen - s), lex_tokss (lexer))
|| lex_look_ahead (lexer) != T_DASH)
return false;
else
diff --git a/src/language/lexer/lexer.h b/src/language/lexer/lexer.h
index 82f3fe2..7af4023 100644
--- a/src/language/lexer/lexer.h
+++ b/src/language/lexer/lexer.h
@@ -93,8 +93,8 @@ char *lex_token_representation (struct lexer *);
/* Token accessors */
enum token_type lex_token (const struct lexer *);
double lex_tokval (const struct lexer *);
-const char *lex_tokid (const struct lexer *);
-const struct string *lex_tokstr (const struct lexer *);
+const char *lex_tokcstr (const struct lexer *);
+struct substring lex_tokss (const struct lexer *);
/* Really weird functions. */
void lex_skip_comment (struct lexer *);
diff --git a/src/language/lexer/q2c.c b/src/language/lexer/q2c.c
index c03b6b2..cf5a04e 100644
--- a/src/language/lexer/q2c.c
+++ b/src/language/lexer/q2c.c
@@ -1542,7 +1542,7 @@ dump_specifier_parse (const specifier *spec, const
subcommand *sbc)
dump (0, "goto lossage;");
dump (-1, "}");
dump (-1, "free (p->%s%s);", sbc->prefix, st_lower (s->valname));
- dump (0, "p->%s%s = xstrdup (ds_cstr (lex_tokstr (lexer)));",
+ dump (0, "p->%s%s = ss_xstrdup (ss_tokss (lexer));",
sbc->prefix, st_lower (s->valname));
}
else
@@ -1705,7 +1705,7 @@ dump_subcommand (const subcommand *sbc)
outdent ();
if (sbc->restriction)
{
- dump (0, "x = ds_length (lex_tokstr (lexer));");
+ dump (0, "x = ss_length (lex_tokss (lexer));");
dump (1, "if (!(%s))", sbc->restriction);
dump (1, "{");
dump (0, "msg (SE, _(\"String for %s must be %s.\"));",
@@ -1715,7 +1715,7 @@ dump_subcommand (const subcommand *sbc)
outdent ();
}
dump (0, "free(p->s_%s);", st_lower(sbc->name) );
- dump (0, "p->s_%s = ds_xstrdup (lex_tokstr (lexer));",
+ dump (0, "p->s_%s = ss_xstrdup (lex_tokss (lexer));",
st_lower (sbc->name));
dump (0, "lex_get (lexer);");
if (sbc->restriction)
@@ -1833,13 +1833,13 @@ dump_parser (int persistent)
{
if (def->type == SBC_VARLIST)
dump (1, "if (lex_token (lexer) == T_ID "
- "&& dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) !=
NULL "
- "&& lex_look_ahead (lexer) != T_EQUALS)");
+ "&& dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) !=
NULL "
+ "&& lex_look_ahead (lexer) != '=')");
else
{
dump (0, "if ((lex_token (lexer) == T_ID "
- "&& dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) "
- "&& lex_look_ahead () != T_EQUALS)");
+ "&& dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) "
+ "&& lex_look_ahead () != '=')");
dump (1, " || token == T_ALL)");
}
dump (1, "{");
diff --git a/src/language/lexer/value-parser.c
b/src/language/lexer/value-parser.c
index 269d24b..4ef092d 100644
--- a/src/language/lexer/value-parser.c
+++ b/src/language/lexer/value-parser.c
@@ -106,7 +106,7 @@ parse_number (struct lexer *lexer, double *x, const enum
fmt_type *format)
assert (fmt_get_category (*format) != FMT_CAT_STRING);
- if (!data_in_msg (ds_ss (lex_tokstr (lexer)), LEGACY_NATIVE,
+ if (!data_in_msg (lex_tokss (lexer), LEGACY_NATIVE,
*format, &v, 0, NULL))
return false;
@@ -143,7 +143,12 @@ parse_value (struct lexer *lexer, union value *v, int
width)
}
else if (lex_force_string (lexer))
{
- const char *s = ds_cstr (lex_tokstr (lexer));
+ const char *s;
+
+ if (!lex_force_string (lexer))
+ return false;
+
+ s = lex_tokcstr (lexer);
value_copy_str_rpad (v, width, CHAR_CAST_BUG (const uint8_t *, s), ' ');
}
else
diff --git a/src/language/lexer/variable-parser.c
b/src/language/lexer/variable-parser.c
index b05b7ce..5b698bb 100644
--- a/src/language/lexer/variable-parser.c
+++ b/src/language/lexer/variable-parser.c
@@ -65,14 +65,14 @@ parse_vs_variable_idx (struct lexer *lexer, const struct
var_set *vs,
lex_error (lexer, _("expecting variable name"));
return false;
}
- else if (var_set_lookup_var_idx (vs, lex_tokid (lexer), idx))
+ else if (var_set_lookup_var_idx (vs, lex_tokcstr (lexer), idx))
{
lex_get (lexer);
return true;
}
else
{
- msg (SE, _("%s is not a variable name."), lex_tokid (lexer));
+ msg (SE, _("%s is not a variable name."), lex_tokcstr (lexer));
return false;
}
}
@@ -339,7 +339,7 @@ parse_var_set_vars (struct lexer *lexer, const struct
var_set *vs,
lex_match (lexer, T_COMMA);
}
while (lex_token (lexer) == T_ALL
- || (lex_token (lexer) == T_ID && var_set_lookup_var (vs, lex_tokid
(lexer)) != NULL));
+ || (lex_token (lexer) == T_ID && var_set_lookup_var (vs, lex_tokcstr
(lexer)) != NULL));
if (*nv == 0)
goto fail;
@@ -444,13 +444,13 @@ parse_DATA_LIST_vars (struct lexer *lexer, char ***names,
lex_error (lexer, "expecting variable name");
goto fail;
}
- if (dict_class_from_id (lex_tokid (lexer)) == DC_SCRATCH
+ if (dict_class_from_id (lex_tokcstr (lexer)) == DC_SCRATCH
&& (pv_opts & PV_NO_SCRATCH))
{
msg (SE, _("Scratch variables not allowed here."));
goto fail;
}
- strcpy (name1, lex_tokid (lexer));
+ strcpy (name1, lex_tokcstr (lexer));
lex_get (lexer);
if (lex_token (lexer) == T_TO)
{
@@ -460,7 +460,7 @@ parse_DATA_LIST_vars (struct lexer *lexer, char ***names,
lex_error (lexer, "expecting variable name");
goto fail;
}
- strcpy (name2, lex_tokid (lexer));
+ strcpy (name2, lex_tokcstr (lexer));
lex_get (lexer);
if (!extract_num (name1, root1, &n1, &d1)
@@ -588,7 +588,7 @@ parse_mixed_vars (struct lexer *lexer, const struct
dictionary *dict,
}
while (lex_token (lexer) == T_ID || lex_token (lexer) == T_ALL)
{
- if (lex_token (lexer) == T_ALL || dict_lookup_var (dict, lex_tokid
(lexer)) != NULL)
+ if (lex_token (lexer) == T_ALL || dict_lookup_var (dict, lex_tokcstr
(lexer)) != NULL)
{
struct variable **v;
size_t nv;
diff --git a/src/language/stats/aggregate.c b/src/language/stats/aggregate.c
index bbe444d..cfa9ea6 100644
--- a/src/language/stats/aggregate.c
+++ b/src/language/stats/aggregate.c
@@ -435,7 +435,7 @@ parse_aggregate_functions (struct lexer *lexer, const
struct dictionary *dict,
if (lex_is_string (lexer))
{
struct string label;
- ds_init_string (&label, lex_tokstr (lexer));
+ ds_init_substring (&label, lex_tokss (lexer));
ds_truncate (&label, 255);
dest_label[n_dest - 1] = ds_xstrdup (&label);
@@ -451,7 +451,7 @@ parse_aggregate_functions (struct lexer *lexer, const
struct dictionary *dict,
goto error;
}
- ds_assign_string (&function_name, lex_tokstr (lexer));
+ ds_assign_substring (&function_name, lex_tokss (lexer));
exclude = ds_chomp (&function_name, '.') ? MV_SYSTEM : MV_ANY;
for (function = agr_func_tab; function->name; function++)
@@ -501,7 +501,7 @@ parse_aggregate_functions (struct lexer *lexer, const
struct dictionary *dict,
lex_match (lexer, T_COMMA);
if (lex_is_string (lexer))
{
- arg[i].c = ds_xstrdup (lex_tokstr (lexer));
+ arg[i].c = ss_xstrdup (lex_tokss (lexer));
type = VAL_STRING;
}
else if (lex_is_number (lexer))
diff --git a/src/language/stats/crosstabs.q b/src/language/stats/crosstabs.q
index 0e7b3da..02cd4de 100644
--- a/src/language/stats/crosstabs.q
+++ b/src/language/stats/crosstabs.q
@@ -381,7 +381,7 @@ crs_custom_tables (struct lexer *lexer, struct dataset *ds,
/* Ensure that this is a TABLES subcommand. */
if (!lex_match_id (lexer, "TABLES")
&& (lex_token (lexer) != T_ID ||
- dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) == NULL)
+ dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) == NULL)
&& lex_token (lexer) != T_ALL)
return 2;
lex_match (lexer, T_EQUALS);
diff --git a/src/language/stats/descriptives.c
b/src/language/stats/descriptives.c
index 17eb445..ae47229 100644
--- a/src/language/stats/descriptives.c
+++ b/src/language/stats/descriptives.c
@@ -334,14 +334,14 @@ cmd_descriptives (struct lexer *lexer, struct dataset *ds)
lex_error (lexer, NULL);
goto error;
}
- if (try_name (dict, dsc, lex_tokid (lexer)))
+ if (try_name (dict, dsc, lex_tokcstr (lexer)))
{
- strcpy (dsc->vars[dsc->var_cnt - 1].z_name, lex_tokid
(lexer));
+ strcpy (dsc->vars[dsc->var_cnt - 1].z_name, lex_tokcstr
(lexer));
z_cnt++;
}
else
msg (SE, _("Z-score variable name %s would be"
- " a duplicate variable name."), lex_tokid
(lexer));
+ " a duplicate variable name."), lex_tokcstr
(lexer));
lex_get (lexer);
if (!lex_force_match (lexer, T_RPAREN))
goto error;
diff --git a/src/language/stats/examine.q b/src/language/stats/examine.q
index db11ef7..aa578e1 100644
--- a/src/language/stats/examine.q
+++ b/src/language/stats/examine.q
@@ -675,7 +675,7 @@ xmn_custom_variables (struct lexer *lexer, struct dataset
*ds,
const struct dictionary *dict = dataset_dict (ds);
lex_match (lexer, T_EQUALS);
- if ( (lex_token (lexer) != T_ID || dict_lookup_var (dict, lex_tokid (lexer))
== NULL)
+ if ( (lex_token (lexer) != T_ID || dict_lookup_var (dict, lex_tokcstr
(lexer)) == NULL)
&& lex_token (lexer) != T_ALL)
{
return 2;
@@ -719,7 +719,7 @@ examine_parse_independent_vars (struct lexer *lexer,
ll_init (&sf->result_list);
if ( (lex_token (lexer) != T_ID ||
- dict_lookup_var (dict, lex_tokid (lexer)) == NULL)
+ dict_lookup_var (dict, lex_tokcstr (lexer)) == NULL)
&& lex_token (lexer) != T_ALL)
{
free ( sf ) ;
@@ -734,7 +734,7 @@ examine_parse_independent_vars (struct lexer *lexer,
lex_match (lexer, T_BY);
if ( (lex_token (lexer) != T_ID ||
- dict_lookup_var (dict, lex_tokid (lexer)) == NULL)
+ dict_lookup_var (dict, lex_tokcstr (lexer)) == NULL)
&& lex_token (lexer) != T_ALL)
{
free (sf);
diff --git a/src/language/stats/frequencies.q b/src/language/stats/frequencies.q
index e397eb8..ecaefdf 100644
--- a/src/language/stats/frequencies.q
+++ b/src/language/stats/frequencies.q
@@ -621,7 +621,7 @@ frq_custom_variables (struct lexer *lexer, struct dataset
*ds,
lex_match (lexer, T_EQUALS);
if (lex_token (lexer) != T_ALL
&& (lex_token (lexer) != T_ID
- || dict_lookup_var (dataset_dict (ds), lex_tokid (lexer)) == NULL))
+ || dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) == NULL))
return 2;
/* Get list of current variables, to avoid duplicates. */
@@ -663,7 +663,8 @@ frq_custom_grouped (struct lexer *lexer, struct dataset
*ds, struct cmd_frequenc
struct frq_proc *frq = frq_;
lex_match (lexer, T_EQUALS);
- if ((lex_token (lexer) == T_ID && dict_lookup_var (dataset_dict (ds),
lex_tokid (lexer)) != NULL)
+ if ((lex_token (lexer) == T_ID
+ && dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer)) != NULL)
|| lex_token (lexer) == T_ID)
for (;;)
{
@@ -739,7 +740,7 @@ frq_custom_grouped (struct lexer *lexer, struct dataset
*ds, struct cmd_frequenc
free (v);
if (!lex_match (lexer, T_SLASH))
break;
- if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds),
lex_tokid (lexer)) != NULL)
+ if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds),
lex_tokcstr (lexer)) != NULL)
&& lex_token (lexer) != T_ALL)
{
lex_put_back (lexer, T_SLASH);
diff --git a/src/language/stats/npar.c b/src/language/stats/npar.c
index b3a24dc..7dd94f7 100644
--- a/src/language/stats/npar.c
+++ b/src/language/stats/npar.c
@@ -277,7 +277,7 @@ parse_npar_tests (struct lexer *lexer, struct dataset *ds,
struct cmd_npar_tests
else if (lex_match_hyphenated_word (lexer, "M-W") ||
lex_match_hyphenated_word (lexer, "MANN-WHITNEY"))
{
- lex_match (lexer, '=');
+ lex_match (lexer, T_EQUALS);
npt->mann_whitney++;
switch (npar_mann_whitney (lexer, ds, nps))
{
@@ -574,7 +574,7 @@ npar_runs (struct lexer *lexer, struct dataset *ds,
nt->execute = runs_execute;
nt->insert_variables = one_sample_insert_variables;
- if ( lex_force_match (lexer, '(') )
+ if ( lex_force_match (lexer, T_LPAREN) )
{
if ( lex_match_id (lexer, "MEAN"))
{
@@ -600,8 +600,8 @@ npar_runs (struct lexer *lexer, struct dataset *ds,
return 0;
}
- lex_force_match (lexer, ')');
- lex_force_match (lexer, '=');
+ lex_force_match (lexer, T_RPAREN);
+ lex_force_match (lexer, T_EQUALS);
if (!parse_variables_const_pool (lexer, specs->pool, dataset_dict (ds),
&tp->vars, &tp->n_vars,
PV_NO_SCRATCH | PV_NO_DUPLICATE | PV_NUMERIC))
@@ -632,7 +632,7 @@ npar_friedman (struct lexer *lexer, struct dataset *ds,
nt->execute = friedman_execute;
nt->insert_variables = one_sample_insert_variables;
- lex_match (lexer, '=');
+ lex_match (lexer, T_EQUALS);
if (!parse_variables_const_pool (lexer, specs->pool, dataset_dict (ds),
&ost->vars, &ost->n_vars,
@@ -663,7 +663,7 @@ npar_kendall (struct lexer *lexer, struct dataset *ds,
nt->execute = friedman_execute;
nt->insert_variables = one_sample_insert_variables;
- lex_match (lexer, '=');
+ lex_match (lexer, T_EQUALS);
if (!parse_variables_const_pool (lexer, specs->pool, dataset_dict (ds),
&ost->vars, &ost->n_vars,
@@ -693,7 +693,7 @@ npar_cochran (struct lexer *lexer, struct dataset *ds,
nt->execute = cochran_execute;
nt->insert_variables = one_sample_insert_variables;
- lex_match (lexer, '=');
+ lex_match (lexer, T_EQUALS);
if (!parse_variables_const_pool (lexer, specs->pool, dataset_dict (ds),
&ft->vars, &ft->n_vars,
@@ -772,7 +772,7 @@ npar_chisquare (struct lexer *lexer, struct dataset *ds,
n = 1;
f = lex_number (lexer);
lex_get (lexer);
- if ( lex_match (lexer, '*'))
+ if ( lex_match (lexer, T_ASTERISK))
{
n = f;
f = lex_number (lexer);
diff --git a/src/language/stats/rank.q b/src/language/stats/rank.q
index 386ef03..f53ef2d 100644
--- a/src/language/stats/rank.q
+++ b/src/language/stats/rank.q
@@ -798,7 +798,7 @@ rank_custom_variables (struct lexer *lexer, struct dataset
*ds, struct cmd_rank
{
lex_match (lexer, T_EQUALS);
- if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds),
lex_tokid (lexer)) == NULL)
+ if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds),
lex_tokcstr (lexer)) == NULL)
&& lex_token (lexer) != T_ALL)
return 2;
@@ -808,7 +808,7 @@ rank_custom_variables (struct lexer *lexer, struct dataset
*ds, struct cmd_rank
if ( lex_match (lexer, T_BY) )
{
- if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds),
lex_tokid (lexer)) == NULL))
+ if ((lex_token (lexer) != T_ID || dict_lookup_var (dataset_dict (ds),
lex_tokcstr (lexer)) == NULL))
{
return 2;
}
@@ -847,9 +847,9 @@ parse_rank_function (struct lexer *lexer, struct dictionary
*dict, struct cmd_ra
while( lex_token (lexer) == T_ID )
{
- if ( dict_lookup_var (dict, lex_tokid (lexer)) != NULL )
+ if ( dict_lookup_var (dict, lex_tokcstr (lexer)) != NULL )
{
- msg(SE, _("Variable %s already exists."), lex_tokid (lexer));
+ msg(SE, _("Variable %s already exists."), lex_tokcstr (lexer));
return 0;
}
if ( var_count >= subcase_get_n_fields (&sc) )
@@ -858,7 +858,7 @@ parse_rank_function (struct lexer *lexer, struct dictionary
*dict, struct cmd_ra
return 0;
}
- destvar = create_rank_variable (dict, f, src_vars[var_count],
lex_tokid (lexer));
+ destvar = create_rank_variable (dict, f, src_vars[var_count],
lex_tokcstr (lexer));
rank_specs[n_rank_specs - 1].destvars[var_count] = destvar ;
lex_get (lexer);
diff --git a/src/language/stats/regression.q b/src/language/stats/regression.q
index 83accc9..e5b86ea 100644
--- a/src/language/stats/regression.q
+++ b/src/language/stats/regression.q
@@ -755,7 +755,7 @@ regression_custom_variables (struct lexer *lexer, struct
dataset *ds,
lex_match (lexer, T_EQUALS);
if ((lex_token (lexer) != T_ID
- || dict_lookup_var (dict, lex_tokid (lexer)) == NULL)
+ || dict_lookup_var (dict, lex_tokcstr (lexer)) == NULL)
&& lex_token (lexer) != T_ALL)
return 2;
diff --git a/src/language/stats/reliability.c b/src/language/stats/reliability.c
index c633197..3633b3d 100644
--- a/src/language/stats/reliability.c
+++ b/src/language/stats/reliability.c
@@ -179,7 +179,7 @@ cmd_reliability (struct lexer *lexer, struct dataset *ds)
if ( ! lex_force_string (lexer) )
goto error;
- ds_init_string (&reliability.scale_name, lex_tokstr (lexer));
+ ds_init_substring (&reliability.scale_name, lex_tokss (lexer));
lex_get (lexer);
diff --git a/src/language/stats/sort-criteria.c
b/src/language/stats/sort-criteria.c
index d808d1c..6497602 100644
--- a/src/language/stats/sort-criteria.c
+++ b/src/language/stats/sort-criteria.c
@@ -94,7 +94,7 @@ parse_sort_criteria (struct lexer *lexer, const struct
dictionary *dict,
}
}
while (lex_token (lexer) == T_ID
- && dict_lookup_var (dict, lex_tokid (lexer)) != NULL);
+ && dict_lookup_var (dict, lex_tokcstr (lexer)) != NULL);
free (local_vars);
return true;
diff --git a/src/language/tests/float-format.c
b/src/language/tests/float-format.c
index 91e4d9e..554b2c3 100644
--- a/src/language/tests/float-format.c
+++ b/src/language/tests/float-format.c
@@ -116,7 +116,7 @@ parse_fp (struct lexer *lexer, struct fp *fp)
|| !lex_force_string (lexer))
return false;
- length = ds_length (lex_tokstr (lexer));
+ length = ss_length (lex_tokss (lexer));
if (fp->format != FLOAT_HEX)
{
if (length != float_get_size (fp->format))
@@ -127,7 +127,7 @@ parse_fp (struct lexer *lexer, struct fp *fp)
return false;
}
assert (length <= sizeof fp->data);
- memcpy (fp->data, ds_data (lex_tokstr (lexer)), length);
+ memcpy (fp->data, ss_data (lex_tokss (lexer)), length);
}
else
{
@@ -136,7 +136,7 @@ parse_fp (struct lexer *lexer, struct fp *fp)
msg (SE, _("Hexadecimal floating constant too long."));
return false;
}
- strncpy (CHAR_CAST_BUG (char *,fp->data), ds_cstr (lex_tokstr
(lexer)), sizeof fp->data);
+ strncpy (CHAR_CAST_BUG (char *,fp->data), lex_tokcstr (lexer),
sizeof fp->data);
}
lex_get (lexer);
diff --git a/src/language/tests/format-guesser-test.c
b/src/language/tests/format-guesser-test.c
index 6285fc0..c95ea35 100644
--- a/src/language/tests/format-guesser-test.c
+++ b/src/language/tests/format-guesser-test.c
@@ -35,8 +35,8 @@ cmd_debug_format_guesser (struct lexer *lexer, struct dataset
*ds UNUSED)
g = fmt_guesser_create ();
while (lex_is_string (lexer))
{
- fprintf (stderr, "\"%s\" ", ds_cstr (lex_tokstr (lexer)));
- fmt_guesser_add (g, ds_ss (lex_tokstr (lexer)));
+ fprintf (stderr, "\"%s\" ", lex_tokcstr (lexer));
+ fmt_guesser_add (g, lex_tokss (lexer));
lex_get (lexer);
}
diff --git a/src/language/tests/paper-size.c b/src/language/tests/paper-size.c
index 5aea2c3..4dbd946 100644
--- a/src/language/tests/paper-size.c
+++ b/src/language/tests/paper-size.c
@@ -1,5 +1,5 @@
/* PSPP - a program for statistical analysis.
- Copyright (C) 2007 Free Software Foundation, Inc.
+ Copyright (C) 2007, 2010 Free Software Foundation, Inc.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -33,7 +33,7 @@ cmd_debug_paper_size (struct lexer *lexer, struct dataset *ds
UNUSED)
if (!lex_force_string (lexer))
return CMD_FAILURE;
- paper_size = ds_cstr (lex_tokstr (lexer));
+ paper_size = lex_tokcstr (lexer);
printf ("\"%s\" => ", paper_size);
if (measure_paper (paper_size, &h, &v))
diff --git a/src/language/utilities/cd.c b/src/language/utilities/cd.c
index 38211e7..220164f 100644
--- a/src/language/utilities/cd.c
+++ b/src/language/utilities/cd.c
@@ -1,5 +1,5 @@
/* PSPP - a program for statistical analysis.
- Copyright (C) 2008 Free Software Foundation, Inc.
+ Copyright (C) 2008, 2010 Free Software Foundation, Inc.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -33,7 +33,7 @@ cmd_cd (struct lexer *lexer, struct dataset *ds UNUSED)
if ( ! lex_force_string (lexer))
goto error;
- path = ds_xstrdup (lex_tokstr (lexer));
+ path = ss_xstrdup (lex_tokss (lexer));
if ( -1 == chdir (path) )
{
diff --git a/src/language/utilities/echo.c b/src/language/utilities/echo.c
index b27b400..149cc0e 100644
--- a/src/language/utilities/echo.c
+++ b/src/language/utilities/echo.c
@@ -34,7 +34,7 @@ cmd_echo (struct lexer *lexer, struct dataset *ds UNUSED)
tab = tab_create(1, 1);
- tab_text(tab, 0, 0, 0, ds_cstr (lex_tokstr (lexer)));
+ tab_text (tab, 0, 0, 0, lex_tokcstr (lexer));
tab_submit(tab);
diff --git a/src/language/utilities/host.c b/src/language/utilities/host.c
index 8610957..f9b1048 100644
--- a/src/language/utilities/host.c
+++ b/src/language/utilities/host.c
@@ -145,7 +145,7 @@ cmd_host (struct lexer *lexer, struct dataset *ds UNUSED)
{
if (!ds_is_empty (&command))
ds_put_byte (&command, '\n');
- ds_put_substring (&command, ds_ss (lex_tokstr (lexer)));
+ ds_put_substring (&command, lex_tokss (lexer));
lex_get (lexer);
}
if (!lex_force_match (lexer, T_RBRACK))
diff --git a/src/language/utilities/include.c b/src/language/utilities/include.c
index d259855..f152684 100644
--- a/src/language/utilities/include.c
+++ b/src/language/utilities/include.c
@@ -170,7 +170,7 @@ cmd_insert (struct lexer *lexer, struct dataset *ds UNUSED)
static int
parse_insert (struct lexer *lexer, char **filename)
{
- char *target_fn;
+ const char *target_fn;
char *relative_filename;
/* Skip optional FILE=. */
@@ -184,7 +184,7 @@ parse_insert (struct lexer *lexer, char **filename)
return CMD_FAILURE;
}
- target_fn = ds_cstr (lex_tokstr (lexer));
+ target_fn = lex_tokcstr (lexer);
relative_filename =
fn_search_path (target_fn,
diff --git a/src/language/utilities/permissions.c
b/src/language/utilities/permissions.c
index 34fa6fd..ad9ffe7 100644
--- a/src/language/utilities/permissions.c
+++ b/src/language/utilities/permissions.c
@@ -50,7 +50,7 @@ cmd_permissions (struct lexer *lexer, struct dataset *ds
UNUSED)
if (!lex_force_string (lexer))
return CMD_FAILURE;
- fn = ds_xstrdup (lex_tokstr (lexer));
+ fn = ss_xstrdup (lex_tokss (lexer));
lex_force_match (lexer, T_STRING);
diff --git a/src/language/utilities/set.q b/src/language/utilities/set.q
index 3883f09..f109e45 100644
--- a/src/language/utilities/set.q
+++ b/src/language/utilities/set.q
@@ -383,27 +383,26 @@ static int
stc_custom_locale (struct lexer *lexer, struct dataset *ds UNUSED,
struct cmd_set *cmd UNUSED, void *aux UNUSED)
{
- const struct string *s;
+ const char *s;
lex_match (lexer, T_EQUALS);
if ( !lex_force_string (lexer))
return 0;
- s = lex_tokstr (lexer);
+ s = lex_tokcstr (lexer);
/* First try this string as an encoding name */
- if ( valid_encoding (ds_cstr (s)))
- set_default_encoding (ds_cstr (s));
+ if ( valid_encoding (s))
+ set_default_encoding (s);
/* Now try as a locale name (or alias) */
- else if (set_encoding_from_locale (ds_cstr (s)))
+ else if (set_encoding_from_locale (s))
{
}
else
{
- msg (ME, _("%s is not a recognized encoding or locale name"),
- ds_cstr (s));
+ msg (ME, _("%s is not a recognized encoding or locale name"), s);
return 0;
}
@@ -511,7 +510,7 @@ stc_custom_journal (struct lexer *lexer, struct dataset *ds
UNUSED, struct cmd_s
journal_disable ();
else if (lex_is_string (lexer) || lex_token (lexer) == T_ID)
{
- journal_set_file_name (ds_cstr (lex_tokstr (lexer)));
+ journal_set_file_name (lex_tokcstr (lexer));
lex_get (lexer);
}
else
diff --git a/src/language/utilities/title.c b/src/language/utilities/title.c
index a3d89b3..82f7be5 100644
--- a/src/language/utilities/title.c
+++ b/src/language/utilities/title.c
@@ -57,7 +57,7 @@ parse_title (struct lexer *lexer, enum text_item_type type)
lex_get (lexer);
if (!lex_force_string (lexer))
return CMD_FAILURE;
- set_title (ds_cstr (lex_tokstr (lexer)), type);
+ set_title (lex_tokcstr (lexer), type);
lex_get (lexer);
return lex_end_of_command (lexer);
}
@@ -149,7 +149,7 @@ cmd_add_documents (struct lexer *lexer, struct dataset *ds)
while ( lex_is_string (lexer))
{
- dict_add_document_line (dict, ds_cstr (lex_tokstr (lexer)));
+ dict_add_document_line (dict, lex_tokcstr (lexer));
lex_get (lexer);
}
diff --git a/src/language/xforms/compute.c b/src/language/xforms/compute.c
index 8245ffe..442c15a 100644
--- a/src/language/xforms/compute.c
+++ b/src/language/xforms/compute.c
@@ -349,10 +349,10 @@ lvalue_parse (struct lexer *lexer, struct dataset *ds)
if (lex_look_ahead (lexer) == T_LPAREN)
{
/* Vector. */
- lvalue->vector = dict_lookup_vector (dict, lex_tokid (lexer));
+ lvalue->vector = dict_lookup_vector (dict, lex_tokcstr (lexer));
if (lvalue->vector == NULL)
{
- msg (SE, _("There is no vector named %s."), lex_tokid (lexer));
+ msg (SE, _("There is no vector named %s."), lex_tokcstr (lexer));
goto lossage;
}
@@ -369,7 +369,7 @@ lvalue_parse (struct lexer *lexer, struct dataset *ds)
else
{
/* Variable name. */
- const char *var_name = lex_tokid (lexer);
+ const char *var_name = lex_tokcstr (lexer);
lvalue->variable = dict_lookup_var (dict, var_name);
if (lvalue->variable == NULL)
{
diff --git a/src/language/xforms/count.c b/src/language/xforms/count.c
index 6d055f8..550db8a 100644
--- a/src/language/xforms/count.c
+++ b/src/language/xforms/count.c
@@ -114,7 +114,7 @@ cmd_count (struct lexer *lexer, struct dataset *ds)
/* Get destination variable, or at least its name. */
if (!lex_force_id (lexer))
goto fail;
- dv->var = dict_lookup_var (dataset_dict (ds), lex_tokid (lexer));
+ dv->var = dict_lookup_var (dataset_dict (ds), lex_tokcstr (lexer));
if (dv->var != NULL)
{
if (var_is_alpha (dv->var))
@@ -124,7 +124,7 @@ cmd_count (struct lexer *lexer, struct dataset *ds)
}
}
else
- dv->name = pool_strdup (trns->pool, lex_tokid (lexer));
+ dv->name = pool_strdup (trns->pool, lex_tokcstr (lexer));
lex_get (lexer);
if (!lex_force_match (lexer, T_EQUALS))
@@ -253,7 +253,7 @@ parse_string_criteria (struct lexer *lexer, struct pool
*pool, struct criteria *
return false;
cur = &crit->values.str[crit->value_cnt++];
*cur = pool_alloc (pool, len + 1);
- str_copy_rpad (*cur, len + 1, ds_cstr (lex_tokstr (lexer)));
+ str_copy_rpad (*cur, len + 1, lex_tokcstr (lexer));
lex_get (lexer);
lex_match (lexer, T_COMMA);
diff --git a/src/language/xforms/recode.c b/src/language/xforms/recode.c
index 12f1eb4..236b498 100644
--- a/src/language/xforms/recode.c
+++ b/src/language/xforms/recode.c
@@ -116,12 +116,12 @@ static bool parse_map_in (struct lexer *lexer, struct
map_in *, struct pool *,
static void set_map_in_generic (struct map_in *, enum map_in_type);
static void set_map_in_num (struct map_in *, enum map_in_type, double, double);
static void set_map_in_str (struct map_in *, struct pool *,
- const struct string *, size_t width);
+ struct substring, size_t width);
static bool parse_map_out (struct lexer *lexer, struct pool *, struct map_out
*);
static void set_map_out_num (struct map_out *, double);
static void set_map_out_str (struct map_out *, struct pool *,
- const struct string *);
+ struct substring);
static void enlarge_dst_widths (struct recode_trns *);
static void create_dst_vars (struct recode_trns *, struct dictionary *);
@@ -318,10 +318,10 @@ parse_map_in (struct lexer *lexer, struct map_in *in,
struct pool *pool,
return false;
else
{
- set_map_in_str (in, pool, lex_tokstr (lexer), max_src_width);
+ set_map_in_str (in, pool, lex_tokss (lexer), max_src_width);
lex_get (lexer);
if (lex_token (lexer) == T_ID
- && lex_id_match (ss_cstr ("THRU"), ss_cstr (lex_tokid (lexer))))
+ && lex_id_match (ss_cstr ("THRU"), lex_tokss (lexer)))
{
msg (SE, _("THRU is not allowed with string variables."));
return false;
@@ -370,13 +370,13 @@ set_map_in_num (struct map_in *in, enum map_in_type type,
double x, double y)
right to WIDTH characters long. */
static void
set_map_in_str (struct map_in *in, struct pool *pool,
- const struct string *string, size_t width)
+ struct substring string, size_t width)
{
in->type = MAP_SINGLE;
value_init_pool (pool, &in->x, width);
value_copy_buf_rpad (&in->x, width,
- CHAR_CAST_BUG (uint8_t *, ds_data (string)),
- ds_length (string), ' ');
+ CHAR_CAST_BUG (uint8_t *, ss_data (string)),
+ ss_length (string), ' ');
}
/* Parses a mapping output value into OUT, allocating memory from
@@ -393,7 +393,7 @@ parse_map_out (struct lexer *lexer, struct pool *pool,
struct map_out *out)
set_map_out_num (out, SYSMIS);
else if (lex_is_string (lexer))
{
- set_map_out_str (out, pool, lex_tokstr (lexer));
+ set_map_out_str (out, pool, lex_tokss (lexer));
lex_get (lexer);
}
else if (lex_match_id (lexer, "COPY"))
@@ -421,10 +421,10 @@ set_map_out_num (struct map_out *out, double value)
/* Sets OUT as a string mapping output with the given VALUE. */
static void
set_map_out_str (struct map_out *out, struct pool *pool,
- const struct string *value)
+ const struct substring value)
{
- const char *string = ds_data (value);
- size_t length = ds_length (value);
+ const char *string = ss_data (value);
+ size_t length = ss_length (value);
if (length == 0)
{
--
1.7.1