Re: Sector file format (was: Re: Testing framework)

[Håkon Løvdal]

On 11/14/05, Patrick Leslie Polzer <address@hidden> wrote:
> If nobody comes up with a better solution, the first and foremost
> requirement is appropriate on-disk storage of specific sectors along
> with device information.
>
> My proposal is this:
>
> 1. Have one file per sector (format: binary octets)
>    Why? This approach has the advantage of easy accessibility with the
>    viewer's favorite hex editor in his favorite format.
>
> 2. Have a master file specifying things like block size and the
>    translation table
>
> 3. Optionally let Parted tar these files (would add an optional
> dependency to libtar and, also optional, gzip or bzip2.
>
> This would require a new source file and header, storage.[hc] with
> functions along these lines (sorted roughly by call order):
>

A tar file consisting of several single sector files could be used, but
I think it should be an absolute requirement that the end user only has
one single, self contained file as output from/input to parted. Anything
else is just begging for trouble I think :)

Below is a prototype of what I had in mind. It is only half complete;
it is currently able to capture everything parted reads from the device,
but it is not yet able to replay it some time later. Keep in mind
that this is intended to be a proof of concept prototype and nothing is
final. Feedback will be appreciated.

Here is a high level summary:

* It has two modes: "record" and "replay".

* Data read from the device is stored (binary) in a "data" file and
  the corresponding meta information in an "index" file (two files
  are currently used for simplicity, in a final version these should
  be merged into one).

* It hooks into the file operation functions in libparted/device.c.
  For "record" mode data will be saved to the "data" file after the real
  read is done, and for "replay" data will be read from the "data" file
  instead of doing a real read.

* Writing is not considered.

* The modifications to device.c are very small, the main work is done
  in a new file which I called filespy.c (but I think this corresponds
  somewhat with what you suggest as storage.c).

* For simplicity I use environmental variables to pass
  arguments to filespy.

* I use a nice and simple (external) library called sl for storing a list
  with the meta information, but it is very simple to install.
  The following command finishes in only half a minute on my machine:
sh -c "wget http://dev.brautaset.org/sl/download/sl-0.3.3.tar.gz \
&& tar zxvf sl-0.3.3.tar.gz && cd sl-0.3.3 \
&& ./configure --prefix=/tmp/sl && make && make install"


Below is a patch along with the index and data file for my /dev/hda attached.


diff -ubBw -r -N parted-1.6.25.1.tar-extract/testfilespy.sh
parted-1.6.25.1/testfilespy.sh
--- parted-1.6.25.1.tar-extract/testfilespy.sh  1970-01-01
01:00:00.000000000 +0100
+++ parted-1.6.25.1/testfilespy.sh      2005-11-15 02:33:41.000000000 +0100
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+rm -f hda.partedcapture-data hda.partedcapture-index
+env FILESPY_MODE=record \
+       env FILESPY_DATA_FILE=hda.partedcapture-data \
+       env FILESPY_INDEX_FILE=hda.partedcapture-index \
+       env FILESPY_INPUT_FILE=/dev/hda \
+       ./parted/parted -s /dev/hda print
+ls -l hda.partedcapture-data hda.partedcapture-index
+cat hda.partedcapture-index
+#rm -f hda.partedcapture-data hda.partedcapture-index
+
diff -ubBw -r -N parted-1.6.25.1.tar-extract/libparted/device.c
parted-1.6.25.1/libparted/device.c
--- parted-1.6.25.1.tar-extract/libparted/device.c      2005-11-11
13:35:53.000000000 +0100
+++ parted-1.6.25.1/libparted/device.c  2005-11-15 02:33:24.000000000 +0100
@@ -30,6 +30,8 @@
 #include <unistd.h>
 #include <errno.h>

+#include "filespy.h"
+
  static PedDevice*     devices; /* legal advice says: initialized to NULL,
                                    under section 6.7.8 part 10
                                    of ISO/EIC 9899:1999 */
@@ -196,7 +198,12 @@
        if (dev->open_count)
                status = ped_architecture->dev_ops->refresh_open (dev);
        else
+       {
                status = ped_architecture->dev_ops->open (dev);
+               if (is_filespy_input(dev->path))
+                       filespy_open();
+       }
+
        if (status)
                dev->open_count++;
        return status;
@@ -212,8 +219,13 @@
        if (--dev->open_count)
                return ped_architecture->dev_ops->refresh_close (dev);
        else
+       {
+               if (is_filespy_input(dev->path))
+                       filespy_close(dev->path, dev->sector_size, dev->length);
+
                return ped_architecture->dev_ops->close (dev);
 }
+}

 /* This function closes a device, while pretending it is still open.  This is
  * useful for temporarily suspending libparted access to the device in order
@@ -253,12 +265,18 @@
  ped_device_read (PedDevice* dev, void* buffer, PedSector start,
                 PedSector count)
 {
+       int ret;
        PED_ASSERT (dev != NULL, return 0);
        PED_ASSERT (buffer != NULL, return 0);
        PED_ASSERT (!dev->external_mode, return 0);
        PED_ASSERT (dev->open_count > 0, return 0);

-       return (ped_architecture->dev_ops->read) (dev, buffer, start, count);
+       if (is_filespy_input(dev->path) && get_filespy_mode() == MODE_REPLAY)
+               return filespy_read_replay(start, count, buffer);
+       ret = (ped_architecture->dev_ops->read) (dev, buffer, start, count);
+       if (is_filespy_input(dev->path) && get_filespy_mode() == MODE_RECORD)
+               filespy_read_record(start, count, buffer);
+       return ret;
 }

 int
diff -ubBw -r -N parted-1.6.25.1.tar-extract/libparted/filespy.h
parted-1.6.25.1/libparted/filespy.h
--- parted-1.6.25.1.tar-extract/libparted/filespy.h     1970-01-01
01:00:00.000000000 +0100
+++ parted-1.6.25.1/libparted/filespy.h 2005-11-15 02:33:24.000000000 +0100
@@ -0,0 +1,18 @@
+#ifndef __FILESPY_H_
+
+#include <parted/device.h>
+
+typedef enum {
+       MODE_RECORD,
+       MODE_REPLAY
+} filespy_mode_t;
+
+filespy_mode_t get_filespy_mode(void);
+int is_filespy_input(char *filename);
+void filespy_open(void);
+void filespy_close(const char *const device_name, int sector_size,
PedSector device_length);
+void filespy_read_record(PedSector start, PedSector count, void *
const buffer);
+int filespy_read_replay(PedSector start, PedSector count, void * const buffer);
+
+#define __FILESPY_H_
+#endif
diff -ubBw -r -N parted-1.6.25.1.tar-extract/libparted/filespy.c
parted-1.6.25.1/libparted/filespy.c
--- parted-1.6.25.1.tar-extract/libparted/filespy.c     1970-01-01
01:00:00.000000000 +0100
+++ parted-1.6.25.1/libparted/filespy.c 2005-11-15 02:33:24.000000000 +0100
@@ -0,0 +1,169 @@
+#include <string.h>
+#define __USE_LARGEFILE        // must define to get ftello from stdio.h
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <parted/geom.h>
+#include <parted/unit.h>
+
+#include <sl/sl.h>
+#include "filespy.h"
+
+#define _ ,
+#define DIE(arg_or_args)   do { fflush(stdout); fprintf(stderr,
"Error: " arg_or_args); exit(1); } while (0)
+
+typedef struct {
+       void *next;
+       PedSector device_sector;   // within "/dev/hda"
+       off_t file_offset;         // within "/tmp/test.hda"
+} node_t;
+
+// DF = device, file
+#define NEW_NODE_DF(_node, _device_sector, _file_offset) \
+do { \
+       _node = malloc(sizeof(node_t)); \
+       _node->device_sector = _device_sector; \
+       _node->file_offset   = _file_offset; \
+} while (0)
+
+typedef struct {
+       int opened;
+       filespy_mode_t mode;
+       FILE *index_stream;
+       FILE *data_stream;
+       node_t *root;
+} filespy_t;
+static filespy_t filespy;
+
+filespy_mode_t get_filespy_mode(void)
+{
+       return filespy.mode;
+}
+
+int is_filespy_input(char *filename)
+{
+       char *input_file = getenv("FILESPY_INPUT_FILE");
+       return (input_file != NULL && strcmp(input_file, filename) == 0);
+}
+
+void filespy_open(void)
+{
+       char *mode;
+       char *index_file;
+       char *data_file;
+
+       if (filespy.opened)
+               return;
+       filespy.opened = 1;
+
+       index_file = getenv("FILESPY_INDEX_FILE");
+       if (index_file == NULL) {
+               DIE("FILESPY_INDEX_FILE not set\n");
+       }
+       data_file = getenv("FILESPY_DATA_FILE");
+       if (data_file == NULL) {
+               DIE("FILESPY_DATA_FILE not set\n");
+       }
+       mode = getenv("FILESPY_MODE");
+       if (mode == NULL) {
+               DIE("FILESPY_MODE not set\n");
+       }
+       if (strcmp(mode, "record") == 0) {
+               filespy.mode = MODE_RECORD;
+       } else if (strcmp(mode, "replay") == 0) {
+               filespy.mode = MODE_REPLAY;
+       } else {
+               DIE("invalid mode value (%s)\n" _ mode);
+       }
+       filespy.index_stream = fopen(index_file, "w");
+       if (filespy.index_stream == NULL) {
+               DIE("failed to open '%s'\n" _ index_file);
+       }
+       filespy.data_stream = fopen(data_file, "w");
+       if (filespy.data_stream == NULL) {
+               DIE("failed to open '%s'\n" _ data_file);
+       }
+       filespy.root = NULL;
+}
+
+
+#define WRITE_STR(str, f) \
+       fwrite(str, strlen(str), 1, f)
+static int write_tree(void * _node, void * _data)
+{
+       char buf[128];
+       (void)_data;
+       const node_t * const node = _node;
+       sprintf(buf, "%lld=%lld\n", node->file_offset, node->device_sector);
+       WRITE_STR(buf, filespy.index_stream);
+       return 0;
+}
+
+#define SPRINTF(_buf, _format, _arg) \
+do { \
+       if (snprintf(_buf, sizeof(_buf), _format, _arg) >= (int)sizeof(_buf)) { 
\
+               _buf[sizeof(_buf)] = '\0'; \
+               DIE(#_buf " too small (%s)\n" _ _buf); \
+       } \
+} while(0)
+
+void filespy_close(const char *const device_name, int sector_size,
PedSector device_length)
+{
+       char buf[1024];
+       WRITE_STR("parted-test-file-v0\n", filespy.index_stream);
+       SPRINTF(buf, "device-name=%s\n", device_name);
+       WRITE_STR(buf, filespy.index_stream);
+       SPRINTF(buf, "device-size=%lld\n", device_length * sector_size);
+       WRITE_STR(buf, filespy.index_stream);
+       SPRINTF(buf, "sector-size=%d\n", sector_size);
+       WRITE_STR(buf, filespy.index_stream);
+       WRITE_STR("#file offset=device sector\n", filespy.index_stream);
+       filespy.root = sl_reverse(filespy.root);
+       sl_map(filespy.root, write_tree, (void*)NULL);
+       sl_free(filespy.root, free);
+       fclose(filespy.index_stream);
+       fclose(filespy.data_stream);
+       memset(&filespy, 0, sizeof(filespy));
+}
+
+static int find_device_sector(void * _node, void * _data)
+{
+       const PedSector * const device_sector_p = _data;
+       const node_t * const node = _node;
+       return node->device_sector == *device_sector_p;
+}
+
+
+void filespy_read_record(PedSector start, PedSector count, void * const buffer)
+{
+       PedSector current_device_sector;
+       current_device_sector = start;
+       do {
+               node_t *node;
+               off_t current_file_offset;
+               current_file_offset = ftello(filespy.data_stream);
+               // fixme: we ought to check that what we read this time is 
identical
+               // with last time for repeated reads
+               if (filespy.root == NULL || sl_map(filespy.root,
find_device_sector, &current_device_sector) == NULL) {
+                       NEW_NODE_DF(node, current_device_sector, 
current_file_offset);
+                       filespy.root = sl_push(filespy.root, node);
+                       fwrite(buffer, PED_SECTOR_SIZE, 1, filespy.data_stream);
+               }
+               count--;
+               current_device_sector++;
+       } while (count > 0);
+}
+
+
+int filespy_read_replay(PedSector start, PedSector count, void * const buffer)
+{
+       PedSector current_device_sector;
+       current_device_sector = start;
+       do {
+               // fixme: not done
+               count--;
+               current_device_sector++;
+       } while (count > 0);
+       return 1;
+}
+

diff -ubBw -r -N parted-1.6.25.1.tar-extract/libparted/Makefile.am
parted-1.6.25.1/libparted/Makefile.am
--- parted-1.6.25.1.tar-extract/libparted/Makefile.am   2005-11-11
13:35:53.000000000 +0100
+++ parted-1.6.25.1/libparted/Makefile.am       2005-11-15 02:33:24.000000000 
+0100
@@ -14,9 +14,9 @@
                fs_ufs                  \
                fs_xfs

-LIBS = @INTLLIBS@ @LIBS@
+LIBS = @INTLLIBS@ @LIBS@ -L/tmp/sl/lib -lsl

-partedincludedir      =        -I$(top_srcdir)/include
+partedincludedir      =        -I$(top_srcdir)/include -I/tmp/sl/include
 lib_LTLIBRARIES              = libparted.la
 libparted_la_LDFLAGS  = -version-info $(LT_CURRENT):$(LT_REVISION):$(LT_AGE) \
                        -release $(LT_RELEASE)
@@ -44,6 +44,7 @@
                        disk_pc98.c             \
                        disk_sun.c              \
                        disk_amiga.c            \
+                       filespy.c               \
                        @address@hidden

 EXTRA_libparted_la_SOURCES    = linux.c                \



BR Håkon Løvdal