2 * Copyright (C) 1999,2000 Erik Walthinsen <omega@cse.ogi.edu>
3 * 2000 Wim Taymans <wtay@chello.be>
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Library General Public
9 * License as published by the Free Software Foundation; either
10 * version 2 of the License, or (at your option) any later version.
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Library General Public License for more details.
17 * You should have received a copy of the GNU Library General Public
18 * License along with this library; if not, write to the
19 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
20 * Boston, MA 02111-1307, USA.
25 #include <sys/types.h>
33 /**********************************************************************
34 * GStreamer Default File Source
37 * This source uses mmap(2) to efficiently load data from a file.
38 * To do this without seriously polluting the applications' memory
39 * space, it must do so in smaller chunks, say 1-4MB at a time.
40 * Buffers are then subdivided from these mmap'd chunks, to directly
41 * make use of the mmap.
43 * To handle refcounting so that the mmap can be freed at the appropriate
44 * time, a buffer will be created for each mmap'd region, and all new
45 * buffers will be sub-buffers of this top-level buffer. As they are
46 * freed, the refcount goes down on the mmap'd buffer and its free()
47 * function is called, which will call munmap(2) on itself.
49 * If a buffer happens to cross the boundaries of an mmap'd region, we
50 * have to decide whether it's more efficient to copy the data into a
51 * new buffer, or mmap() just that buffer. There will have to be a
52 * breakpoint size to determine which will be done. The mmap() size
53 * has a lot to do with this as well, because you end up in double-
54 * jeopardy: the larger the outgoing buffer, the more data to copy when
55 * it overlaps, *and* the more frequently you'll have buffers that *do*
58 * Seeking is another tricky aspect to do efficiently. The initial
59 * implementation of this source won't make use of these features, however.
60 * The issue is that if an application seeks backwards in a file, *and*
61 * that region of the file is covered by an mmap that hasn't been fully
62 * deallocated, we really should re-use it. But keeping track of these
63 * regions is tricky because we have to lock the structure that holds
64 * them. We need to settle on a locking primitive (GMutex seems to be
65 * a really good option...), then we can do that.
69 GstElementDetails gst_filesrc_details = {
72 "Read from arbitrary point in a file",
74 "Erik Walthinsen <omega@cse.ogi.edu>",
78 //#define fs_print(format,args...) g_print(format, ## args)
79 #define fs_print(format,args...)
82 #define GST_TYPE_FILESRC \
83 (gst_filesrc_get_type())
84 #define GST_FILESRC(obj) \
85 (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_FILESRC,GstFileSrc))
86 #define GST_FILESRC_CLASS(klass) \
87 (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_FILESRC,GstFileSrcClass))
88 #define GST_IS_FILESRC(obj) \
89 (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_FILESRC))
90 #define GST_IS_FILESRC_CLASS(obj) \
91 (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_FILESRC))
94 GST_FILESRC_OPEN = GST_ELEMENT_FLAG_LAST,
96 GST_FILESRC_FLAG_LAST = GST_ELEMENT_FLAG_LAST + 2,
99 typedef struct _GstFileSrc GstFileSrc;
100 typedef struct _GstFileSrcClass GstFileSrcClass;
106 guint pagesize; // system page size
108 gchar *filename; // filename
109 gint fd; // open file descriptor
110 off_t filelen; // what's the file length?
112 off_t curoffset; // current offset in file
113 off_t block_size; // bytes per read
114 gboolean touch; // whether to touch every page
120 GMutex *map_regions_lock;
122 gboolean seek_happened;
125 struct _GstFileSrcClass {
126 GstElementClass parent_class;
130 /* FileSrc signals and args */
148 static void gst_filesrc_class_init (GstFileSrcClass *klass);
149 static void gst_filesrc_init (GstFileSrc *filesrc);
151 static void gst_filesrc_set_property (GObject *object, guint prop_id,
152 const GValue *value, GParamSpec *pspec);
153 static void gst_filesrc_get_property (GObject *object, guint prop_id,
154 GValue *value, GParamSpec *pspec);
156 static GstBuffer * gst_filesrc_get (GstPad *pad);
157 static gboolean gst_filesrc_srcpad_event (GstPad *pad, GstEvent *event);
159 static GstElementStateReturn gst_filesrc_change_state (GstElement *element);
162 static GstElementClass *parent_class = NULL;
163 //static guint gst_filesrc_signals[LAST_SIGNAL] = { 0 };
166 gst_filesrc_get_type(void)
168 static GType filesrc_type = 0;
171 static const GTypeInfo filesrc_info = {
172 sizeof(GstFileSrcClass), NULL,
174 (GClassInitFunc)gst_filesrc_class_init,
179 (GInstanceInitFunc)gst_filesrc_init,
181 filesrc_type = g_type_register_static (GST_TYPE_ELEMENT, "GstFileSrc", &filesrc_info, 0);
187 gst_filesrc_class_init (GstFileSrcClass *klass)
189 GObjectClass *gobject_class;
190 GstElementClass *gstelement_class;
192 gobject_class = (GObjectClass*)klass;
193 gstelement_class = (GstElementClass*)klass;
195 parent_class = g_type_class_ref (GST_TYPE_ELEMENT);
197 g_object_class_install_property(G_OBJECT_CLASS(klass), ARG_LOCATION,
198 g_param_spec_string("location","File Location","Location of the file to read",
199 NULL,G_PARAM_READWRITE));
200 g_object_class_install_property(G_OBJECT_CLASS(klass), ARG_FILESIZE,
201 g_param_spec_int64("filesize","File Size","Size of the file being read",
202 0,G_MAXINT64,0,G_PARAM_READABLE));
203 g_object_class_install_property(G_OBJECT_CLASS(klass), ARG_FD,
204 g_param_spec_int("fd","File-descriptor","File-descriptor for the file being read",
205 0,G_MAXINT,0,G_PARAM_READABLE));
206 g_object_class_install_property(G_OBJECT_CLASS(klass), ARG_BLOCKSIZE,
207 g_param_spec_ulong("blocksize","Block Size","Block size to read per buffer",
208 0,G_MAXULONG,4096,G_PARAM_READWRITE));
209 g_object_class_install_property(G_OBJECT_CLASS(klass), ARG_OFFSET,
210 g_param_spec_int64("offset","File Offset","Byte offset of current read pointer",
211 0,G_MAXINT64,0,G_PARAM_READWRITE));
212 g_object_class_install_property(G_OBJECT_CLASS(klass), ARG_MAPSIZE,
213 g_param_spec_ulong("mmapsize","mmap() Block Size","Size in bytes of mmap()d regions",
214 0,G_MAXULONG,4*1048576,G_PARAM_READWRITE));
215 g_object_class_install_property(G_OBJECT_CLASS(klass), ARG_TOUCH,
216 g_param_spec_boolean("touch","Touch read data","Touch data to force disk read before push()",
217 TRUE,G_PARAM_READWRITE));
219 gobject_class->set_property = gst_filesrc_set_property;
220 gobject_class->get_property = gst_filesrc_get_property;
222 gstelement_class->change_state = gst_filesrc_change_state;
226 gst_filesrc_bufcmp (gconstpointer a, gconstpointer b)
228 // GstBuffer *bufa = (GstBuffer *)a, *bufb = (GstBuffer *)b;
230 // sort first by offset, then in reverse by size
231 if (GST_BUFFER_OFFSET(a) < GST_BUFFER_OFFSET(b)) return -1;
232 else if (GST_BUFFER_OFFSET(a) > GST_BUFFER_OFFSET(b)) return 1;
233 else if (GST_BUFFER_SIZE(a) > GST_BUFFER_SIZE(b)) return -1;
234 else if (GST_BUFFER_SIZE(a) < GST_BUFFER_SIZE(b)) return 1;
239 gst_filesrc_init (GstFileSrc *src)
241 src->srcpad = gst_pad_new ("src", GST_PAD_SRC);
242 gst_pad_set_get_function (src->srcpad,gst_filesrc_get);
243 gst_pad_set_event_function (src->srcpad,gst_filesrc_srcpad_event);
244 gst_element_add_pad (GST_ELEMENT (src), src->srcpad);
246 src->pagesize = getpagesize();
248 src->filename = NULL;
253 src->block_size = 4096;
257 src->mapsize = 4 * 1024 * 1024; // default is 4MB
259 src->map_regions = g_tree_new(gst_filesrc_bufcmp);
260 src->map_regions_lock = g_mutex_new();
262 src->seek_happened = FALSE;
267 gst_filesrc_set_property (GObject *object, guint prop_id, const GValue *value, GParamSpec *pspec)
271 /* it's not null if we got it, but it might not be ours */
272 g_return_if_fail (GST_IS_FILESRC (object));
274 src = GST_FILESRC (object);
278 /* the element must be stopped in order to do this */
279 g_return_if_fail (GST_STATE (src) < GST_STATE_PLAYING);
281 if (src->filename) g_free (src->filename);
282 /* clear the filename if we get a NULL (is that possible?) */
283 if (g_value_get_string (value) == NULL) {
284 gst_element_set_state (GST_ELEMENT (object), GST_STATE_NULL);
285 src->filename = NULL;
286 /* otherwise set the new filename */
288 src->filename = g_strdup (g_value_get_string (value));
292 src->block_size = g_value_get_ulong (value);
295 src->curoffset = g_value_get_int64 (value);
298 if ((src->mapsize % src->pagesize) == 0)
299 src->mapsize = g_value_get_ulong (value);
301 GST_INFO(0, "invalid mapsize, must a multiple of pagesize, which is %d\n",src->pagesize);
304 src->touch = g_value_get_boolean (value);
312 gst_filesrc_get_property (GObject *object, guint prop_id, GValue *value, GParamSpec *pspec)
316 /* it's not null if we got it, but it might not be ours */
317 g_return_if_fail (GST_IS_FILESRC (object));
319 src = GST_FILESRC (object);
323 g_value_set_string (value, src->filename);
326 g_value_set_int64 (value, src->filelen);
329 g_value_set_int (value, src->fd);
332 g_value_set_ulong (value, src->block_size);
335 g_value_set_int64 (value, src->curoffset);
338 g_value_set_ulong (value, src->mapsize);
341 g_value_set_boolean (value, src->touch);
344 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
350 gst_filesrc_free_parent_mmap (GstBuffer *buf)
352 GstFileSrc *src = GST_FILESRC(GST_BUFFER_POOL_PRIVATE(buf));
354 fs_print ("freeing mmap()d buffer at %d+%d\n",GST_BUFFER_OFFSET(buf),GST_BUFFER_SIZE(buf));
356 // remove the buffer from the list of available mmap'd regions
357 g_mutex_lock(src->map_regions_lock);
358 g_tree_remove(src->map_regions,buf);
359 // check to see if the tree is empty
360 if (g_tree_nnodes(src->map_regions) == 0) {
361 // we have to free the bufferpool we don't have yet
363 g_mutex_unlock(src->map_regions_lock);
366 // madvise to tell the kernel what to do with it
367 madvise(GST_BUFFER_DATA(buf),GST_BUFFER_SIZE(buf),MADV_DONTNEED);
369 // now unmap the memory
370 munmap(GST_BUFFER_DATA(buf),GST_BUFFER_MAXSIZE(buf));
374 gst_filesrc_map_region (GstFileSrc *src, off_t offset, size_t size)
379 g_return_val_if_fail (offset >= 0, NULL);
381 fs_print ("mapping region %08lx+%08lx from file into memory\n",offset,size);
383 // time to allocate a new mapbuf
384 buf = gst_buffer_new();
385 // mmap() the data into this new buffer
386 GST_BUFFER_DATA(buf) = mmap (NULL, size, PROT_READ, MAP_SHARED, src->fd, offset);
387 if (GST_BUFFER_DATA(buf) == NULL) {
388 fprintf (stderr, "ERROR: gstfilesrc couldn't map file!\n");
389 } else if (GST_BUFFER_DATA(buf) == MAP_FAILED) {
390 g_error ("gstfilesrc mmap(0x%x, %d, 0x%llx) : %s",
391 size, src->fd, offset, sys_errlist[errno]);
393 #ifdef MADV_SEQUENTIAL
394 // madvise to tell the kernel what to do with it
395 retval = madvise(GST_BUFFER_DATA(buf),GST_BUFFER_SIZE(buf),MADV_SEQUENTIAL);
397 // fill in the rest of the fields
398 GST_BUFFER_FLAGS(buf) = GST_BUFFER_READONLY | GST_BUFFER_ORIGINAL;
399 GST_BUFFER_SIZE(buf) = size;
400 GST_BUFFER_MAXSIZE(buf) = size;
401 GST_BUFFER_OFFSET(buf) = offset;
402 GST_BUFFER_TIMESTAMP(buf) = -1LL;
403 GST_BUFFER_POOL_PRIVATE(buf) = src;
404 GST_BUFFER_FREE_FUNC(buf) = gst_filesrc_free_parent_mmap;
406 g_mutex_lock(src->map_regions_lock);
407 g_tree_insert(src->map_regions,buf,buf);
408 g_mutex_unlock(src->map_regions_lock);
414 gst_filesrc_map_small_region (GstFileSrc *src, off_t offset, size_t size)
420 // printf("attempting to map a small buffer at %d+%d\n",offset,size);
422 // if the offset starts at a non-page boundary, we have to special case
423 if ((mod = offset % src->pagesize)) {
426 mapbase = offset - mod;
427 mapsize = ((size + mod + src->pagesize - 1) / src->pagesize) * src->pagesize;
428 // printf("not on page boundaries, resizing map to %d+%d\n",mapbase,mapsize);
429 map = gst_filesrc_map_region(src, mapbase, mapsize);
430 ret = gst_buffer_create_sub (map, offset - mapbase, size);
432 gst_buffer_unref (map);
437 return gst_filesrc_map_region(src,offset,size);
445 // This allows us to search for a potential mmap region.
447 gst_filesrc_search_region_match (gpointer a, gpointer b)
449 GstFileSrcRegion *r = (GstFileSrcRegion *)b;
451 // trying to walk b down the tree, current node is a
452 if (r->offset < GST_BUFFER_OFFSET(a)) return -1;
453 else if (r->offset >= (GST_BUFFER_OFFSET(a) + GST_BUFFER_SIZE(a))) return 1;
454 else if ((r->offset + r->size) <= (GST_BUFFER_OFFSET(a) + GST_BUFFER_SIZE(a))) return 0;
461 * @pad: #GstPad to push a buffer from
463 * Push a new buffer from the filesrc at the current offset.
466 gst_filesrc_get (GstPad *pad)
469 GstBuffer *buf = NULL, *map;
471 off_t readend,mapstart,mapend;
472 GstFileSrcRegion region;
475 g_return_val_if_fail (pad != NULL, NULL);
476 src = GST_FILESRC (gst_pad_get_parent (pad));
477 g_return_val_if_fail (GST_FLAG_IS_SET (src, GST_FILESRC_OPEN), NULL);
480 if (src->seek_happened) {
481 src->seek_happened = FALSE;
482 return GST_BUFFER (gst_event_new(GST_EVENT_DISCONTINUOUS));
486 if (src->curoffset == src->filelen) {
487 gst_element_set_state (GST_ELEMENT (src), GST_STATE_PAUSED);
488 return GST_BUFFER (gst_event_new(GST_EVENT_EOS));
491 // calculate end pointers so we don't have to do so repeatedly later
492 readsize = src->block_size;
493 readend = src->curoffset + src->block_size; // note this is the byte *after* the read
494 mapstart = GST_BUFFER_OFFSET(src->mapbuf);
495 mapend = mapstart + GST_BUFFER_SIZE(src->mapbuf); // note this is the byte *after* the map
497 // check to see if we're going to overflow the end of the file
498 if (readend > src->filelen) {
499 readsize = src->filelen - src->curoffset;
500 readend = src->curoffset;
503 // if the start is past the mapstart
504 if (src->curoffset >= mapstart) {
505 // if the end is before the mapend, the buffer is in current mmap region...
506 // ('cause by definition if readend is in the buffer, so's readstart)
507 if (readend <= mapend) {
508 fs_print ("read buf %d+%d lives in current mapbuf %d+%d, creating subbuffer of mapbuf\n",
509 src->curoffset,readsize,GST_BUFFER_OFFSET(src->mapbuf),GST_BUFFER_SIZE(src->mapbuf));
510 buf = gst_buffer_create_sub (src->mapbuf, src->curoffset - GST_BUFFER_OFFSET(src->mapbuf),
513 // if the start actually is within the current mmap region, map an overlap buffer
514 } else if (src->curoffset < mapend) {
515 fs_print ("read buf %d+%d starts in mapbuf %d+%d but ends outside, creating new mmap\n",
516 src->curoffset,readsize,GST_BUFFER_OFFSET(src->mapbuf),GST_BUFFER_SIZE(src->mapbuf));
517 buf = gst_filesrc_map_small_region (src, src->curoffset, readsize);
520 // the only other option is that buffer is totally outside, which means we search for it
522 // now we can assume that the start is *before* the current mmap region
523 // if the readend is past mapstart, we have two options
524 } else if (readend >= mapstart) {
525 // either the read buffer overlaps the start of the mmap region
526 // or the read buffer fully contains the current mmap region
527 // either way, it's really not relevant, we just create a new region anyway
528 fs_print ("read buf %d+%d starts before mapbuf %d+%d, but overlaps it\n",
529 src->curoffset,readsize,GST_BUFFER_OFFSET(src->mapbuf),GST_BUFFER_SIZE(src->mapbuf));
530 buf = gst_filesrc_map_small_region (src, src->curoffset, readsize);
533 // then deal with the case where the read buffer is totally outside
535 // first check to see if there's a map that covers the right region already
536 fs_print ("searching for mapbuf to cover %d+%d\n",src->curoffset,readsize);
537 region.offset = src->curoffset;
538 region.size = readsize;
539 map = g_tree_search (src->map_regions,
540 (GCompareFunc) gst_filesrc_search_region_match,
543 // if we found an exact match, subbuffer it
545 fs_print ("found mapbuf at %d+%d, creating subbuffer\n",GST_BUFFER_OFFSET(map),GST_BUFFER_SIZE(map));
546 buf = gst_buffer_create_sub (map, src->curoffset - GST_BUFFER_OFFSET(map), readsize);
548 // otherwise we need to create something out of thin air
550 // if the read buffer crosses a mmap region boundary, create a one-off region
551 if ((src->curoffset / src->mapsize) != (readend / src->mapsize)) {
552 fs_print ("read buf %d+%d crosses a %d-byte boundary, creating a one-off\n",
553 src->curoffset,readsize,src->mapsize);
554 buf = gst_filesrc_map_small_region (src, src->curoffset, readsize);
556 // otherwise we will create a new mmap region and set it to the default
558 off_t nextmap = src->curoffset - (src->curoffset % src->mapsize);
559 fs_print ("read buf %d+%d in new mapbuf at %d+%d, mapping and subbuffering\n",
560 src->curoffset,readsize,nextmap,src->mapsize);
561 // first, we're done with the old mapbuf
562 gst_buffer_unref(src->mapbuf);
564 src->mapbuf = gst_filesrc_map_region (src, nextmap, src->mapsize);
566 buf = gst_buffer_create_sub (src->mapbuf, src->curoffset - GST_BUFFER_OFFSET(src->mapbuf), readsize);
571 /* if we need to touch the buffer (to bring it into memory), do so */
573 for (i=0;i<GST_BUFFER_SIZE(buf);i+=src->pagesize)
574 *(GST_BUFFER_DATA(buf)+i) = *(GST_BUFFER_DATA(buf)+i);
577 /* we're done, return the buffer */
578 src->curoffset += GST_BUFFER_SIZE(buf);
582 /* open the file and mmap it, necessary to go to READY state */
584 gst_filesrc_open_file (GstFileSrc *src)
586 g_return_val_if_fail (!GST_FLAG_IS_SET (src ,GST_FILESRC_OPEN), FALSE);
588 GST_DEBUG(0, "opening file %s\n",src->filename);
591 src->fd = open (src->filename, O_RDONLY);
594 gst_element_error (GST_ELEMENT (src), g_strconcat("opening file \"", src->filename, "\"", NULL));
597 /* find the file length */
598 src->filelen = lseek (src->fd, 0, SEEK_END);
599 lseek (src->fd, 0, SEEK_SET);
601 // allocate the first mmap'd region
602 src->mapbuf = gst_filesrc_map_region (src, 0, src->mapsize);
606 GST_FLAG_SET (src, GST_FILESRC_OPEN);
611 /* unmap and close the file */
613 gst_filesrc_close_file (GstFileSrc *src)
615 g_return_if_fail (GST_FLAG_IS_SET (src, GST_FILESRC_OPEN));
621 /* zero out a lot of our state */
626 GST_FLAG_UNSET (src, GST_FILESRC_OPEN);
630 static GstElementStateReturn
631 gst_filesrc_change_state (GstElement *element)
633 g_return_val_if_fail (GST_IS_FILESRC (element), GST_STATE_FAILURE);
635 if (GST_STATE_PENDING (element) == GST_STATE_NULL) {
636 if (GST_FLAG_IS_SET (element, GST_FILESRC_OPEN))
637 gst_filesrc_close_file (GST_FILESRC (element));
639 if (!GST_FLAG_IS_SET (element, GST_FILESRC_OPEN)) {
640 if (!gst_filesrc_open_file (GST_FILESRC (element)))
641 return GST_STATE_FAILURE;
645 if (GST_ELEMENT_CLASS (parent_class)->change_state)
646 return GST_ELEMENT_CLASS (parent_class)->change_state (element);
648 return GST_STATE_SUCCESS;
652 gst_filesrc_srcpad_event (GstPad *pad, GstEvent *event)
654 GstFileSrc *src = GST_FILESRC(GST_PAD_PARENT(pad));
656 switch (GST_EVENT_TYPE (event)) {
658 src->curoffset = (guint64) GST_EVENT_SEEK_OFFSET (event);
659 src->seek_happened = TRUE;
660 gst_event_free (event);
661 // push a discontinuous event?