Content-type: text/html Man page of GFAL


Section: Library Functions (3)
Updated: $Date: 2009/04/16 13:54:45 $
Index Return to Main Contents


gfal - Grid File Access Library



 enum se_type {TYPE_NONE = 0, TYPE_SRM, TYPE_SRMv2, TYPE_SE};

 typedef struct gfal_filestatus {
    char *surl;
    char *turl;
    int status;
    char *explanation;
    int    pinlifetime;
    struct stat64 stat;
    struct gfal_filestatus *subpaths;
    int nbsubpaths;
 } gfal_filestatus;

 typedef struct gfal_request {
    /* [optional]
     * if set to 1, 'surls' to NULL, and endpoint specified,
     * 'nbfiles' SURLs will be generated */
    int generatesurls;

    /* [optional]
     * only used to generate SURLs, if specified */
    char *relative_path;

    /* [required]
     * number of files in 'surls' */
    int nbfiles;

    /* [required]
     * list of surls */
    char **surls;

    /* [optional]
     * endpoint */
    char *endpoint;
    /* [required by gfal_turlsfromsurls]
     * 0 for get request, 1 for put request */
    int oflag;
    /* [optional]
     * list of file sizes
     * can be useful for gfal_turlsfromsurls with put request */
    GFAL_LONG64 *filesizes;
    /* [optional]
     * SE type to use *by default* */
    enum se_type defaultsetype;
    /* [optional]
     * SE type to use - will fail if this type cannot be used */
    enum se_type setype;
    /* [optional]
     * if set to 1, no bdii call will be done
     * endpoint must be set up for SRM-compliant SE */
    int no_bdii_check;

    /* [optional]
     * timeout */
    int timeout;

    /* [optional]
     * list of protocols to use */
    char **protocols;

    /* [optional] only used with SRMv2 */
    char *srmv2_spacetokendesc;
    int srmv2_desiredpintime;
    int srmv2_lslevels;
    int srmv2_lsoffset;
    int srmv2_lscount;
 } *gfal_request;

 /******************** Common functions ********************/
 const char *gfal_version ();
 int gfal_set_verbose (int);
 int gfal_set_vo (const char *vo);
 void gfal_set_nobdii (int);
 void gfal_set_timeout_connect (int);
 int gfal_get_timeout_connect ();
 void gfal_set_timeout_sendreceive (int);
 int gfal_get_timeout_sendreceive ();
 void gfal_set_timeout_bdii (int);
 int gfal_get_timeout_bdii ();
 void gfal_set_timeout_srm (int);
 int gfal_get_timeout_srm ();
 /******************** POSIX functions ********************/
 int gfal_access (const char *, int);
 int gfal_chmod (const char *, mode_t);
 int gfal_close (int);
 int gfal_closedir (DIR *);
 int gfal_creat (const char *, mode_t);
 int gfal_creat64 (const char *, mode_t);
 off_t gfal_lseek (int, off_t, int);
 int gfal_mkdir (const char *, mode_t);
 int gfal_open (const char *, int, mode_t);
 int gfal_open64 (const char *, int, mode_t);
 DIR *gfal_opendir (const char *);
 ssize_t gfal_read (int, void *, size_t);
 struct dirent *gfal_readdir (DIR *);
 int gfal_rename (const char *, const char *);
 int gfal_rmdir (const char *);
 ssize_t gfal_setfilchg (int, const void *, size_t);
 int gfal_unlink (const char *);
 ssize_t gfal_write (int, const void *, size_t);
 off64_t gfal_lseek64 (int, off64_t, int);
 int gfal_lstat (const char *, struct stat *);
 int gfal_lstat64 (const char *, struct stat64 *);
 int gfal_stat (const char *, struct stat *);
 int gfal_stat64 (const char *, struct stat64 *);
 struct dirent64 *gfal_readdir64 (DIR *);
 /******************** SRM functions ********************/
 gfal_request gfal_request_new ();
 int gfal_init (gfal_request, gfal_internal *, char *, int);
 int gfal_deletesurls (gfal_internal, char *, int);
 int gfal_removedir (gfal_internal, char *, int);
 int gfal_turlsfromsurls (gfal_internal, char *, int);
 int gfal_ls (gfal_internal, char *, int);
 int gfal_ls_end (gfal_internal, char *, int);
 int gfal_get (gfal_internal, char *, int);
 int gfal_getstatus (gfal_internal, char *, int);
 int gfal_prestage (gfal_internal, char *, int);
 int gfal_prestagestatus (gfal_internal, char *, int);
 int gfal_pin (gfal_internal, char *, int);
 int gfal_release (gfal_internal, char *, int);
 int gfal_set_xfer_done (gfal_internal, char *, int);
 int gfal_set_xfer_running (gfal_internal, char *, int);
 int gfal_abortrequest (gfal_internal, char *, int);
 int gfal_abortfiles (gfal_internal, char *, int);
 int gfal_get_results (gfal_internal, gfal_filestatus **);
 int gfal_get_ids_setype (gfal_internal, enum se_type *, int *, int **, char **);
 int gfal_get_ids (gfal_internal, int *, int **, char **);
 int gfal_set_ids (gfal_internal, int, const int *, int, const char *, char *, int);
 void gfal_internal_free (gfal_internal);



Grid storage interactions today require using several existing software components:
The replica catalog services to locate valid replicas of files.
The SRM software to ensure:
files exist on disk (they are recalled from mass storage if necessary) or
space is allocated on disk for new files (they are possibly migrated to mass storage later)
A file access mechanism to access files from the storage system on the worker node.

The GFAL library hides these interactions and presents a Posix interface for the I/O operations. The currently supported protocols are: file for local access, dcap, gsidcap and kdcap (dCache access protocol) and rfio (CASTOR/DPM access protocol).

The function names are obtained by prepending gfal_ to the Posix names, for example gfal_open, gfal_read, gfal_close ...

The argument lists and the values returned by the functions are identical.

File naming convention
A file name can be a Logical File Name (LFN), a Grid Unique IDentifier (GUID), a file replica (SURL) or a Transport file name (TURL).

an LFN starts with lfn:

for example lfn:baud/testgfal15

a GUID starts with guid:

for example guid:2cd59291-7ae7-4778-af6d-b1f423719441

an SURL starts with srm://

for example srm://

a TURL starts with a protocol name

for example rfio:////castor/
Compile and link
The header file gfal_api.h needs to be included in the application source code to get the prototype of the functions. The application must be linked with libgfal (a few auxiliary security libraries: libcgsi_plugin_gsoap_2.3, libglobus_gssapi_gsi_gcc32dbg and libglobus_gss_assist_gcc32dbg are used internally).

All these libraries are available as shared libraries as well as archive libraries.

BDII endpoint
A BDII stores a lot of data concerning, like SE type (SRMv1, SRMv2.2, Classic SE, ...), default endpoints for VOs (LFC, RLS, ...). So, one must specify the BDII to use with the environment variable LCG_GFAL_INFOSYS. For example :


Virtual Organization - VO
To be able to get endpoints from BDII (in case they are not specified elsewhere, see below), or to select the rigth default SE from environment variables (see below), one must specify the environment variable LCG_GFAL_VO. For example :

    setenv LCG_GFAL_VO dteam

EDG Replica Catalogs endpoints
To avoid GFAL to get them from BDII (slower), one can specify the servers endpoints with environment variables RMC_ENDPOINT and LRC_ENDPOINT. If they aren't specified, a default value will be get from BDII. To use such File Catalog, one must set the LCG_CATALOG_TYPE variable to "edg". For example :

    setenv RMC_ENDPOINT \

    setenv LRC_ENDPOINT \

LCG File Catalog - LFC
An alternative to the EDG Replica Catalogs is to use the LCG File Catalog (LFC) instead. In order to use the LFC, one must set the LCG_CATALOG_TYPE variable to "lfc" and optionally provide the LFC_HOST to specify which host to contact. If it is not specified, a default value will be get from BDII. For example:

    setenv LCG_CATALOG_TYPE lfc
and optionally

    setenv LFC_HOST

Keep fully qualified SURLs in file catalog
By default, before storing a SURL in file catalog, SRM-version-specific information is removed. That way, SURLs are generic enough to match all SRM servers, BDII being used to get missing information. When BDII is not used at all in a project, full SURLs need to be stored directly in file catalog. In such a case, LCG_GFAL_FULL_SURLS_IN_FC must be defined to whatever value:

       setenv LCG_GFAL_FULL_SURLS_IN_FC 1

But, be CAREFUL, defining that when BDII can be used will prevent users from choosing the SRM version they want to use, and will create complexity when SRM version will change!

RFIO protocol
For the moment there are 2 different versions of RFIO protocol : Castor and DPM. It is planned to merge them into a single common version, but it isn't available yet. By default, the Castor version is used. So, if you want to use the DPM version, one must set LCG_RFIO_TYPE variable to "dpm". For example :

    setenv LCG_RFIO_TYPE dpm

The SRMs can only be accessed in secure mode, i.e. require a valid Grid certificate and valid Grid proxy obtained by grid-proxy-init. The Replica catalogs can be accessed in secure or insecure mode depending on the endpoint provided: if it starts with https: the secure mode is used.



Posix GFAL function usage is standard, so we won't spend time to explain it. To have more details on a specific function, please se the corresponding man page. Due to the 'gfal_internal' object use by most of other GFAL functions, to use them requires to follow a few steps.

First of all, you have to create a 'gfal_request' object, in which you will be able to set your parameters :

    gfal_request req;
    if ((req = gfal_request_new ()) == NULL) {
        // ERROR ...
    req->surls = ...
    req->nbfiles = ...

To see the meaning of each parameter, have a look to the documentation of gfal_init(). Once, you have set all the desired parameters in this request, you have to the internal gfal object like this :

    gfal_internal gfal;
    if (gfal_init (req, &gfal, errbuf, errbufsz) < 0) {
        // ERROR ...

DO NOT MODIFY this object by hand. If you need other parameters, just make a new request and run gfal_init again on it. Moreover, do not free any members of req while you are using the internal object, because it uses them !

Then, you just have to pass this internal object gfal to gfal functions :

    if (gfal_turlsfromsurls (gfal, errbuf, errbufsz) < 0) {
        // ERROR ...

Finally, you have to call a specific function to get the results :

    gfal_filestatus *filestatuses;
    if (gfal_get_results (gfal, &filestatuses) < 0) {
        // ERROR ...

It is recommended to keep the same 'gfal_internal' object (one per thread) for making requests on same SURLs.

Once the 'gfal_internal' object is not needed anymore, it must be freed:

    gfal_internal_free (gfal);

But, it won't free the arguments which were in your initial request (eg. list of SURLs), you have to free them by hand if needed, in addition to the request ('gfal_request' object).



Some cases may require to have more debug information to understand what is going on, especially what information is really exchanged between GFAL and other components.

It is possible to get this information by setting environment variables.

For bash:
    export CGSI_TRACE=1
    export CGSI_TRACEFILE=/path/to/your/file

For csh:
    setenv CGSI_TRACE 1
    sentenv CGSI_TRACEFILE /path/to/your/file

Then just run your program, and information which is exchanged between GFAL and other components will be put into /path/to/your/file



All function return 0 on success, and -1 on failure. The variable errno is set to the Posix error codes in the case of failure.



A program writing a file, reading it back and checking the content of the file is shown below:
#include <fcntl.h>
#include <stdio.h>
#include "gfal_api.h"
#define BLKLEN 65536

main(int argc, char **argv)
    int fd;
    int i;
    char ibuf[BLKLEN];
    char obuf[BLKLEN];
    int rc;

    if (argc != 2) {
        fprintf (stderr, "usage: %s filename\n", argv[0]);
        exit (1);

    for (i = 0; i < BLKLEN; i++)
        obuf[i] = i;

    printf ("creating file %s\n", argv[1]);
    if ((fd = gfal_open (argv[1], O_WRONLY|O_CREAT, 0644)) < 0) {
        perror ("gfal_open");
        exit (1);
    printf ("open successful, fd = %d\n", fd);

    if ((rc = gfal_write (fd, obuf, BLKLEN)) != BLKLEN) {
        if (rc < 0)
            perror ("gfal_write");
            fprintf (stderr, "gfal_write returns %d\n", rc);
        (void) gfal_close (fd);
        exit (1);
    printf ("write successful\n");

    if ((rc = gfal_close (fd)) < 0) {
        perror ("gfal_close");
        exit (1);
    printf ("close successful\n");

    printf ("reading back %s\n", argv[1]);
    if ((fd = gfal_open (argv[1], O_RDONLY, 0)) < 0) {
        perror ("gfal_open");
        exit (1);
    printf ("open successful, fd = %d\n", fd);

    if ((rc = gfal_read (fd, ibuf, BLKLEN)) != BLKLEN) {
        if (rc < 0)
            perror ("gfal_read");
            fprintf (stderr, "gfal_read returns %d\n", rc);
        (void) gfal_close (fd);
        exit (1);
    printf ("read successful\n");

    if ((rc = gfal_close (fd)) < 0) {
        perror ("gfal_close");
        exit (1);
    printf ("close successful\n");

    for (i = 0; i < BLKLEN; i++) {
        if (ibuf[i] != obuf[i]) {
            fprintf (stderr, "compare failed at offset %d\n", i);
            exit (1);
    printf ("compare successful\n");
    exit (0);






gfal(3), gfal_request_new(3), gfal_internal_free(3), gfal_init(3), gfal_deletesurls(3), gfal_removedir(3), gfal_turlsfromsurls(3), gfal_ls(3), gfal_get(3), gfal_getstatus(3), gfal_prestage(3), gfal_prestagestatus(3), gfal_pin(3), gfal_release(3), gfal_abortrequest(3), gfal_abortfiles(3), gfal_set_xfer_done(3), gfal_set_xfer_running(3), gfal_get_ids(3), gfal_set_ids(3), gfal_get_results(3), gfal_get_timeout_connect(3), gfal_get_timeout_sendreceive(3), gfal_get_timeout_bdii(3), gfal_get_timeout_srm(3), gfal_set_timeout_connect(3), gfal_set_timeout_sendreceive(3), gfal_set_timeout_bdii(3), gfal_set_timeout_srm(3), gfal_set_verbose(3),




This document was created by man2html, using the manual pages.
Time: 14:16:31 GMT, July 14, 2009