Re: proposal, funopen for libsox

[Jan Stary <hans@stare.cz>]

On May 20 18:07:24, ole.hellqvist@gmail.com wrote:
> Hi all
> For BSD's the the memstream functions don't work.

Can you elaborate please?

I can only see two "memstream functions":

./src/example5.c:  assert(out = sox_open_memstream_write(&buffer, &buffer_size, &in->signal, NULL, "sox", NULL));
./src/formats.c:        buffer_ptr? open_memstream(buffer_ptr, buffer_size_ptr) :
./src/formats.c:sox_format_t * sox_open_memstream_write( ./src/sox.h:Opens an encoding session for a memstream buffer. Returned handle must be closed with sox_close().
./src/sox.h:sox_open_memstream_write(

The sox_open_memstream_write() function
is defined in formats.c and used in example5.c

The use of open_memstream() in formats.c is #ifdef'd as

	ft->fp =
	#ifdef HAVE_FMEMOPEN
              buffer? fmemopen(buffer, buffer_size, "w+b") :
	              buffer_ptr? open_memstream(buffer_ptr,
		      buffer_size_ptr) :
	#endif


OpenBSD does have HAVE_FMEMOPEN
(indeed, config.log says so on my OpenBSD current/i386);

example5 works fine on this system, as in
./example5 monkey.wav out.wav && play out.wav

So what exactly does not work for you?

> I have made a patch for openbsd that seems to solve this.
> The files involved are formats.c and configure.ac.
> For configure.ac  it is only one line to change:
> "
>  dnl Checks for library functions.
> -AC_CHECK_FUNCS(strcasecmp strdup popen vsnprintf gettimeofday mkstemp fmemopen)
> +AC_CHECK_FUNCS(strcasecmp strdup popen vsnprintf gettimeofday mkstemp fmemopen funopen)
> "
> For formats.c the patch is larger, see below.
> So far I have verified this patch with "example5.c"
> in both modes.
> openbsd patch for  formats.c (sox version 14.4.0) :

What I wrote above uses 14.4.1; can you try with 14.4.1,
and preferably with the current git too?
Do you see the same problem that you have with 14.4.0?


	Jan

> $OpenBSD$
> --- src/formats.c.orig	Sun Mar  4 23:54:16 2012
> +++ src/formats.c	Mon May 20 15:48:34 2013
> @@ -36,7 +36,229 @@
>  #if HAVE_MAGIC
>    #include <magic.h>
>  #endif
> +#ifdef HAVE_FUNOPEN 
> +#include <stdio.h>
> +struct fmem {
> +    size_t pos;
> +    size_t size;
> +    char *buffer;
> +};
> +typedef struct fmem fmem_t;
>  
> +static int readfn(void *handler, char *buf, int size) {
> +  fmem_t *mem = handler;
> +  size_t available = mem->size - mem->pos;
> +
> +    if ((size > (int) (mem->size - mem->pos)) && ( mem->size >=  mem->pos))  {
> +    size = available;
> +  }
> +  memcpy(buf, mem->buffer + mem->pos, sizeof(char) * size);
> +  mem->pos += size;
> +  
> +  return size;
> +}
> +
> +static int writefn(void *handler, const char *buf, int size) {
> +  fmem_t *mem = handler;
> +  size_t available = mem->size - mem->pos;
> +
> +   if ((size > (int) (mem->size - mem->pos)) && ( mem->size >=  mem->pos))  {
> +    size = available;
> +  }
> +  memcpy(mem->buffer + mem->pos, buf, sizeof(char) * size);
> +  mem->pos += size;
> +
> +  return size;
> +}
> +
> +static fpos_t seekfn(void *handler, fpos_t offset, int whence) {
> +  size_t pos;
> +  fmem_t *mem = handler;
> +
> +  switch (whence) {
> +    case SEEK_SET: pos = offset; break;
> +    case SEEK_CUR: pos = mem->pos + offset; break;
> +    case SEEK_END: pos = mem->size + offset; break;
> +    default: return -1;
> +  }
> +
> +  if (pos > mem->size) {
> +    return -1;
> +  }
> +
> +  mem->pos = pos;
> +  return (fpos_t)pos;
> +}
> +
> +static int closefn(void *handler) {
> +  free(handler);
> +  return 0;
> +}
> +
> +
> +/*memstream :--------------------------------------- 
> +*extracted from: The Android Open Source Project
> +*
> +* Licensed under the Apache License, Version 2.0 
> +
> +* Definition of a seekable, write-only memory stream.
> +*/
> +typedef struct {
> +    char** bufp; /* pointer to buffer pointer */
> +    size_t* sizep; /* pointer to eof */
> +
> +    size_t allocSize; /* size of buffer */
> +    size_t eof; /* furthest point we've written to */
> +    size_t offset; /* current write offset */
> +    char saved; /* required by NUL handling */
> +} MemStream;
> +typedef struct MemStream  memstream_t;
> +#define kInitialSize 1024
> +
> +#if 0
> +# define DBUG(x) printf x
> +#else
> +# define DBUG(x) ((void)0)
> +#endif
> +/*
> +* Ensure that we have enough storage to write "size" bytes at the
> +* current offset. We also have to take into account the extra '\0'
> +* that we maintain just past EOF.
> +*
> +* Returns 0 on success.
> +*/
> +static int ensureCapacity(MemStream* stream, int writeSize)
> +{
> +    size_t newSize;
> +    size_t neededSize;
> +    char* newBuf;
> +    DBUG(("+++ ensureCap off=%d size=%d\n", stream->offset, writeSize));
> +
> +     neededSize = stream->offset + writeSize + 1;
> +    if (neededSize <= stream->allocSize)
> +        return 0;
> +
> +    if (stream->allocSize == 0) {
> +        newSize = kInitialSize;
> +    } else {
> +        newSize = stream->allocSize;
> +        newSize += newSize / 2; /* expand by 3/2 */
> +    }
> +
> +    if (newSize < neededSize)
> +        newSize = neededSize;
> +    DBUG(("+++ realloc %p->%p to size=%d\n",
> +        stream->bufp, *stream->bufp, newSize));
> +     newBuf = (char*) realloc(*stream->bufp, newSize);
> +    if (newBuf == NULL)
> +        return -1;
> +
> +    *stream->bufp = newBuf;
> +    stream->allocSize = newSize;
> +    return 0;
> +}
> +
> +/*
> +* Write data to a memstream, expanding the buffer if necessary.
> +*
> +* If we previously seeked beyond EOF, zero-fill the gap.
> +*
> +* Returns the number of bytes written.
> +*/
> +static int write_memstream(void* cookie, const char* buf, int size)
> +{
> +    MemStream* stream = (MemStream*) cookie;
> +
> +    if (ensureCapacity(stream, size) < 0)
> +        return -1;
> +
> +    /* seeked past EOF earlier? */
> +    if (stream->eof < stream->offset) {
> +        DBUG(("+++ zero-fill gap from %d to %d\n",
> +            stream->eof, stream->offset-1));
> +        memset(*stream->bufp + stream->eof, '\0',
> +            stream->offset - stream->eof);
> +    }
> +
> +    /* copy data, advance write pointer */
> +    memcpy(*stream->bufp + stream->offset, buf, (size_t) size);
> +    stream->offset += size;
> +
> +    if (stream->offset > stream->eof) {
> +        /* EOF has advanced, update it and append null byte */
> +        DBUG(("+++ EOF advanced to %d, appending nul\n", stream->offset));
> +        assert(stream->offset < stream->allocSize);
> +        stream->eof = stream->offset;
> +    } else {
> +        /* within previously-written area; save char we're about to stomp */
> +        DBUG(("+++ within written area, saving '%c' at %d\n",
> +            *(*stream->bufp + stream->offset), stream->offset));
> +        stream->saved = *(*stream->bufp + stream->offset);
> +    }
> +    *(*stream->bufp + stream->offset) = '\0';
> +    *stream->sizep = stream->offset;
> +
> +    return size;
> +}
> +
> +/*
> +* Seek within a memstream.
> +*
> +* Returns the new offset, or -1 on failure.
> +*/
> +static fpos_t seek_memstream(void* cookie, fpos_t offset, int whence)
> +{
> +    MemStream* stream = (MemStream*) cookie;
> +    off_t newPosn = (off_t) offset;
> +
> +    if (whence == SEEK_CUR) {
> +        newPosn += stream->offset;
> +    } else if (whence == SEEK_END) {
> +        newPosn += stream->eof;
> +    }
> +
> +    if (newPosn < 0 || ((fpos_t)((size_t) newPosn)) != newPosn) {
> +        /* bad offset - negative or huge */
> +        DBUG(("+++ bogus seek offset %ld\n", (long) newPosn));
> +        errno = EINVAL;
> +        return (fpos_t) -1;
> +    }
> +
> +    if (stream->offset < stream->eof) {
> +        /*
> +* We were pointing to an area we'd already written to, which means
> +* we stomped on a character and must now restore it.
> +*/
> +        DBUG(("+++ restoring char '%c' at %d\n",
> +            stream->saved, stream->offset));
> +        *(*stream->bufp + stream->offset) = stream->saved;
> +    }
> +
> +    stream->offset = (size_t) newPosn;
> +
> +    if (stream->offset < stream->eof) {
> +        /*
> +* We're seeked backward into the stream. Preserve the character
> +* at EOF and stomp it with a NUL.
> +*/
> +        stream->saved = *(*stream->bufp + stream->offset);
> +        *(*stream->bufp + stream->offset) = '\0';
> +        *stream->sizep = stream->offset;
> +    } else {
> +        /*
> +* We're positioned at, or possibly beyond, the EOF. We want to
> +* publish the current EOF, not the current position.
> +*/
> +        *stream->sizep = stream->eof;
> +    }
> +
> +    return newPosn;
> +}
> +
> +
> +
> +#endif
> +
>  #define AUTO_DETECT_SIZE 4096
>  
>  static char const * auto_detect_format(sox_format_t * ft, char const * ext)
> @@ -406,8 +628,14 @@ static void UNUSED rewind_pipe(FILE * fp)
>  /* _FSTDIO is for Torek stdio (i.e. most BSD-derived libc's)
>   * In theory, we no longer need to check _NEWLIB_VERSION or __APPLE__ */
>  #if defined _FSTDIO || defined _NEWLIB_VERSION || defined __APPLE__
> +#  ifdef __DragonFly__
> +  struct __FILE_public *fpp = (struct __FILE_public *)fp;
> +  fpp->_p -= AUTO_DETECT_SIZE;
> +  fpp->_r += AUTO_DETECT_SIZE;
> +#  else
>    fp->_p -= AUTO_DETECT_SIZE;
>    fp->_r += AUTO_DETECT_SIZE;
> +#  endif
>  #elif defined __GLIBC__
>    fp->_IO_read_ptr = fp->_IO_read_base;
>  #elif defined _MSC_VER || defined __MINGW_H || defined _ISO_STDIO_ISO_H
> @@ -456,16 +684,30 @@ static sox_format_t * open_read(
>        ft->fp = stdin;
>      }
>      else {
> -      ft->fp =
> +#ifdef HAVE_FUNOPEN
> +	fmem_t *memr = (fmem_t *) malloc(sizeof(fmem_t));
> +	memset(memr, 0, sizeof(fmem_t));
> +	memr->size = buffer_size ;
> +	memr->buffer = buffer;
> +#endif	
> +      ft->fp = 
>  #ifdef HAVE_FMEMOPEN
> -        buffer? fmemopen(buffer, buffer_size, "rb") :
> +        (buffer != NULL)? fmemopen(buffer, buffer_size, "rb") :
>  #endif
> -        xfopen(path, "rb", &ft->io_type);
> -      type = io_types[ft->io_type];
> +#ifdef HAVE_FUNOPEN
> +        (buffer != NULL)? funopen(memr, readfn, writefn, seekfn, closefn) :
> +#endif
> +		xfopen(path, "rb", &ft->io_type);
> +     	type = io_types[ft->io_type];
>        if (ft->fp == NULL) {
> +#ifdef HAVE_FUNOPEN
> +        lsx_fail("can't open FUNOPEN  %s `%s': %s", type, path, strerror(errno));
> +#else
>          lsx_fail("can't open input %s `%s': %s", type, path, strerror(errno));
> -        goto error;
> +#endif
> +	  goto error;
>        }
> +
>      }
>      if (setvbuf (ft->fp, NULL, _IOFBF, sizeof(char) * input_bufsiz)) {
>        lsx_fail("Can't set read buffer");
> @@ -825,7 +1067,26 @@ static sox_format_t * open_write(
>  {
>    sox_format_t * ft = lsx_calloc(sizeof(*ft), 1);
>    sox_format_handler_t const * handler;
> +#ifdef HAVE_FUNOPEN
> +  fmem_t *memw;
> +  MemStream*  ms;
> +    if (buffer != NULL)
> +    {
> +	memw = (fmem_t *) malloc(sizeof(fmem_t));
> +	memset(memw, 0, sizeof(fmem_t));
> +	memw->size = buffer_size ;
> +	memw->buffer = buffer ;
>  
> +    }
> +  else if  (buffer_ptr != NULL)
> +  { 
> +      ms = (MemStream*) calloc((size_t) 1, sizeof(MemStream));
> +
> +      ms->bufp = buffer_ptr;
> +      ms->sizep = &buffer_size;
> +  }
> +#endif
> +
>    if (!path || !signal) {
>      lsx_fail("must specify file name and signal parameters to write file");
>      goto error;
> @@ -853,12 +1114,17 @@ static sox_format_t * open_write(
>          lsx_fail("permission to overwrite `%s' denied", path);
>          goto error;
>        }
> +
>        ft->fp =
>  #ifdef HAVE_FMEMOPEN
> -        buffer? fmemopen(buffer, buffer_size, "w+b") :
> -        buffer_ptr? open_memstream(buffer_ptr, buffer_size_ptr) :
> +	    (buffer != NULL) ? fmemopen(buffer, buffer_size, "w+b") :
> +        (buffer_ptr != NULL) ? open_memstream(buffer_ptr, buffer_size_ptr) :
>  #endif
> -        fopen(path, "w+b");
> +#ifdef HAVE_FUNOPEN
> +        (buffer != NULL)? funopen(memw, readfn, writefn, seekfn, closefn) :
> +	(buffer_ptr != NULL) ? funopen(ms, NULL ,write_memstream, seek_memstream, closefn ):
> +#endif
> +	    fopen(path, "w+b");
>        if (ft->fp == NULL) {
>          lsx_fail("can't open output file `%s': %s", path, strerror(errno));
>          goto error;
> ----------------------------
> best regards /Ole
> 
> 
>  
> 
> 
> 
> 
> 
> 
> 
> 
> -- 

------------------------------------------------------------------------------
AlienVault Unified Security Management (USM) platform delivers complete
security visibility with the essential security capabilities. Easily and
efficiently configure, manage, and operate all of your security controls
from a single console and one unified framework. Download a free trial.
http://p.sf.net/sfu/alienvault_d2d