Adapting existing programs to large files

If you wish to adapt existing programs to access large files, you must follow the procedures that are described in the section "Creating new programs", i.e. add the appropriate Define statements and the Include file unistd.h, replace the 32-bit functions (open(), lseek(), ...) with the corresponding 64-Bit functions open64(), lseek64(), ... and if necessary use the 64-bit data types. This may mean internal data structures may have to be modified in an incompatible way.

To clarify the problems associated with accessing large files, a small program prog32 is given below as an example. This program is subsequently converted so that it can access large files (Program example prog64 in "Adapting existing programs to large files").

Program example prog32

prog32 opens a specified file and outputs a maximum of 32 Bytes of this file starting from a specified point. For reasons of legibility, the input parameters check has been omitted.

/*
** prog.c
** 
** Parameter 1: Name of file to be read
** Parameter 2: Offset of data to be read in file
** 
*/
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/fcntl.h>
#define BUFFER_LENGTH   8192
#define READ_LENGTH     32
char buffer[BUFFER_LENGTH];
int
main(int argc, char *argv[]) {
int fd;
int len;
int i;
off_t filelen;
off_t offset_in_file;
   offset_in_file = atol(argv[2]);
   printf ("reading from file <%s> with offset %d and length %d\n", argv[1], 
         offset_in_file, READ_LENGTH);
   /* open file */
   if ((fd = open (argv[1], O_RDONLY)) < 0) {
      printf ("open not successful, termination\n");
      perror("ERRNO SET");
      exit(EXIT_FAILURE);
   }
   /* now get the length of the file */
   if ((filelen = lseek(fd, (off_t)0, SEEK_END)) == (off_t)(-1)) {
      printf ("lseek to end of file not successful, termination\n");
      perror("ERRNO SET");
      exit(EXIT_FAILURE);
   }
   if (offset_in_file > filelen) {
      printf ("offset %d is greater than filelength %d, termination\n",
            offset_in_file, filelen);
      exit(EXIT_FAILURE);
   }
   /* now seek to the offset to be read */
   if (lseek(fd, offset_in_file - filelen, SEEK_CUR) == (off_t)(-1)) {
      printf ("lseek not successful, termination\n");
      perror("ERRNO SET");
      exit(EXIT_FAILURE);
   }
   /* read the data */
   if ((len = read (fd, &buffer[0], READ_LENGTH)) <= 0 ) {
      printf ("read not successful, termination\n");
      perror("ERRNO SET");
      exit(EXIT_FAILURE);
   }
   else {
      /* now print the data that were read in hexadecimal form */
      printf ("data of size %d (expected %d) were read\n", len, READ_LENGTH);
      for (i = 0; i < len; i++) {
         printf ("%02X ", buffer[i]);
      }
      printf ("\n");
   }
   if (close(fd) != 0) {
      printf ("close not successful, termination\n");
      perror("ERRNO SET");
      exit(EXIT_FAILURE);
   }
}

Program example prog64

prog64 is derived from prog32 and converted so that it can access large files.

/*
** prog.c
**
** 1. Parameter: Name of file to be read
** 2. Parameter: Offset of data to be read in file
**
*/
#define _LARGEFILE64_SOURCE   1
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/fcntl.h>
#define BUFFER_LENGTH        8192
#define READ_LENGTH          32
char buffer[BUFFER_LENGTH];
int
main(int argc, char *argv[]) {
int fd;
int len;
int i;
off64_t filelen;
off64_t offset_in_file;
   offset_in_file = atoll(argv[2]);
   printf ("reading from file <%s> with offset %lld and length %ld\n",
             argv[1], offset_in_file, READ_LENGTH);
   /* open file */
   if ((fd = open64 (argv[1], O_RDONLY)) < 0) {
      printf ("open not successful, termination\n");
      perror("ERRNO SET");
      exit(EXIT_FAILURE);
   }
   /* now get the length of the file */
   if ((filelen = lseek64(fd, (off64_t)0, SEEK_END)) == (off64_t)(-1)) {
      printf ("lseek to end of file not successful, termination\n");
      perror("ERRNO SET");
      exit(EXIT_FAILURE);
   }
   if (offset_in_file > filelen) {
      printf ("offset %lld is greater than filelength %lld, termination\n",
            offset_in_file, filelen);
      exit(EXIT_FAILURE);
   }
   /* now seek to the offset to be read */
   if (lseek64(fd, offset_in_file - filelen, SEEK_CUR) == (off64_t)(-1)) {
      printf ("lseek not successful, termination\n");
      perror("ERRNO SET");
      exit(EXIT_FAILURE);
   }
   /* read the data */
   if ((len = read (fd, &buffer[0], READ_LENGTH)) <= 0 ) {
      printf ("read not successful, termination\n");
      perror("ERRNO SET");
      exit(EXIT_FAILURE);
   }
   else {
      /* now print the data that were read in hexadecimal form */
      printf ("data of size %d (expected %d) were read\n", len, READ_LENGTH);
      for (i = 0; i < len; i++) {
         printf ("%02X ", buffer[i]);
      }
      printf ("\n");
   }
   if (close(fd) != 0) {
      printf ("close not successful, termination\n");
      perror("ERRNO SET");
      exit(EXIT_FAILURE);
   }
}

Applying prog32 and prog64

Both programs were applied to the following files in the /mnt33/dir1 directory:

$ ls -l /mnt33/dir1
total 10245088
-rw-r--r--   1 BACH     OS315    2621440000 Feb 27 18:26 bigfile1
-rw-r--r--   1 BACH     OS315    2621440000 Mar  6 15:06 bigfile2
-rw-r--r--   1 BACH     OS315         18 Mar 15 13:56 smallfile1
-rw-r--r--   1 BACH     OS315         26 Mar 15 13:57 smallfile2
drwxr-xr-x   2 BACH     OS315       2048 Mar 15 13:58 subdir1
drwxr-xr-x   2 BACH     OS315       2048 Mar 15 13:58 subdir2

Results of some program runs:

$ prog32 /mnt33/dir1/smallfile1  128
reading from file </mnt33/dir1/smallfile1> with offset 128 and length 32
offset 128 is greater than filelength 18, termination
$ prog32 /mnt33/dir1/bigfile1  128  28
reading from file </mnt33/dir1/bigfile1> with offset 128 and length 32
lseek to end of file not successful, termination
ERRNO SET: Value too large for defined data type
$ prog64 /mnt33/dir1/smallfile1  128
reading from file </mnt33/dir1/smallfile1> with offset 128 and length 32
offset 128 is greater than filelength 18, termination

$ prog64 /mnt33/dir1/smallfile1  10 28
reading from file </mnt33/dir1/smallfile1> with offset 10 and length 32
data of size 8 (expected 32) were read
91 A5 A2 A5 A2 A5 95 15 
$ prog64 /mnt33/dir1/bigfile1  128  0
reading from file </mnt33/dir1/bigfile1> with offset 128 and length 32
data of size 32 (expected 32) were read
F3 F3 F3 F3 F3 F3 F3 F3 F3 F3 F3 F3 F3 F3 F3 F3 F3 F3 F3 F3 F3 F3 F3 F3 F3 F3 
F3 F3 F3 F3 F3 F3 
$ prog64 /mnt33/dir1/bigfile1  2500000000
reading from file </mnt33/dir1/bigfile1> with offset 2500000000 and length 32
data of size 32 (expected 32) were read
F1 F1 F1 F1 F1 F1 F1 F1 F1 F1 F1 F1 F1 F1 F1 F1 F1 F1 F1 F1 F1 F1 F1 F1 F1 F1 
F1 F1 F1 F1 F1 F1 
$ prog64 /mnt33/dir1/bigfile1  500000000000
reading from file </mnt33/dir1/bigfile1> with offset 5000000000 and length 32
offset 5000000000 is greater than filelength 2621440000, termination