Unix Programming - problem with implementing pipe

Author

problem with implementing pipe

Greg

2004-09-29, 3:09 am

I am trying to implement the UNIX pipe command using C but with the
"->" operator. Everything works fine with 1 pipe, but when I try to
use 2 or more, it hangs up when reading the pipe_in filestream. If
ANYONE could offer ANY suggestion as to why this is happening it would
be much appreciated.

Thanks in advance!

#define MAX_PIPES 5
#define MAX_CMD_LEN 255

#include <string.h>
#include <stdio.h>

// global vars

char piped_cmds [MAX_PIPES][MAX_CMD_LEN] ;
int numPipes ;

// fcn prototypes

int pipeCmds (int cmd1, int cmd2) ;
char *stripFirstChar (char *str) ;
char *stripLastChar (char *str) ;
void splitPipes (char *toSplit) ;
int isPipe (char *cmd) ;

int main () {
int i ;
char command [] = "ps aux -> grep greg -> wc" ;
//char command [] = "cat .bash_profile -> tail -15 -> tail -10 ->
tail -5 -> tail -3" ;
//char command [] = "ps aux -> grep greg -> grep 5 -> grep 3 -> less"
;
char *noPipe = "ps aux" ;

//numPipes = 0 ;
if (isPipe (command)) {
splitPipes (command) ;
for (i = 0; i < numPipes - 1; i++) {
if (!pipeCmds (i, i + 1))
printf ("Pipe failed\n") ;
else
printf ("Pipe succeeded\n") ;

}
}
printf ("Exit\n") ;
}

int pipeCmds (int cmd1, int cmd2) {
FILE *pipe_in ;
FILE *pipe_out ;
char *my_string ;
int nbytes = 100;
int bytes_read;
char readbuf[MAX_CMD_LEN];

printf ("Piping [%i] %s -> [%i] %s\n", cmd1, piped_cmds [cmd1], cmd2,
piped_cmds [cmd2]) ;
// open pipes
pipe_in = popen (piped_cmds [cmd1], "r") ;
printf ("Input pipe open\n") ;
pipe_out = popen (piped_cmds [cmd2], "w") ;
printf ("Output pipe open\n") ;

if ((!pipe_in) || (!pipe_out)) {
fprintf (stderr, "One or both pipes failed.\n") ;
return 0 ;
}

printf ("Transferring file contents...\n") ;
// while(fgets(readbuf, 250, pipe_in))
// fputs(readbuf, pipe_out);
while (!feof (pipe_in)) {
printf ("Reading input pipe...\n") ;
/****************program hangs here**************************/
fgets (readbuf, MAX_CMD_LEN, pipe_in);
if(feof(pipe_in))
break;
printf("%s", readbuf) ;
fputs (readbuf, pipe_out) ;
}

pclose (pipe_in) ;
pclose (pipe_out) ;
printf ("Pipes closed\n") ;

return 1 ;

}

char *stripFirstChar (char *str) {
str++ ;
return str ;
}

char *stripLastChar (char *str) {
char *temp = str ;
while (*temp != '\0') {
temp++ ;
}
temp-- ;
*temp = '\0' ;
//printf ("\"%s\"\n", temp) ;

return str ;
}

void splitPipes (char *toSplit) {
char *tokenPtr = NULL ;
int i ;

for (tokenPtr = strtok (toSplit, "->"); tokenPtr != NULL;
tokenPtr = strtok (NULL, "->"))
{
strcpy (piped_cmds [numPipes], tokenPtr) ;
numPipes++ ;
}

for (i = 1; i < numPipes; i++) {
strcpy (piped_cmds [i], stripFirstChar (piped_cmds [i])) ;
}

for (i = 0; i < numPipes - 1; i++) {
strcpy (piped_cmds [i], stripLastChar (piped_cmds [i])) ;
}

for (i = 0; i < numPipes; i++) {
printf ("[%i] \"%s\"\n", i, piped_cmds [i]) ;
}
}

int isPipe (char *cmd) {

if (strchr (cmd, '>')) {
printf ("%s contains at least 1 pipe\n", cmd) ;
return 1 ;
}
else
printf ("%s contains no pipes\n", cmd) ;
return 0 ;
}

Barry Margolin

2004-09-29, 3:09 am

In article <7e312ede.0409282245.1273003f@posting.google.com>,
djbitchpimp@snowboard.com (Greg) wrote:

> I am trying to implement the UNIX pipe command using C but with the
> "->" operator. Everything works fine with 1 pipe, but when I try to
> use 2 or more, it hangs up when reading the pipe_in filestream. If
> ANYONE could offer ANY suggestion as to why this is happening it would
> be much appreciated.

When you have 3 commands in the pipe, you're running piped_cmds[1]
twice: first when you call pipeCmds(0, 1), then again when you call
pipeCmds(1, 2). And when you run it the second time, the command is
trying to read from your standard input rather than the output of the
first command.

Another problem with your design is that the third command in the
pipeline isn't even started until the first command finishes. If the
first command is an infinite loop (like "tail -f filename"), the third
command will never be run.

>
> Thanks in advance!
>
>
>
> #define MAX_PIPES 5
> #define MAX_CMD_LEN 255
>
> #include <string.h>
> #include <stdio.h>
>
> // global vars
>
> char piped_cmds [MAX_PIPES][MAX_CMD_LEN] ;
> int numPipes ;
>
> // fcn prototypes
>
> int pipeCmds (int cmd1, int cmd2) ;
> char *stripFirstChar (char *str) ;
> char *stripLastChar (char *str) ;
> void splitPipes (char *toSplit) ;
> int isPipe (char *cmd) ;
>
> int main () {
> int i ;
> char command [] = "ps aux -> grep greg -> wc" ;
> //char command [] = "cat .bash_profile -> tail -15 -> tail -10 ->
> tail -5 -> tail -3" ;
> //char command [] = "ps aux -> grep greg -> grep 5 -> grep 3 -> less"
> ;
> char *noPipe = "ps aux" ;
>
> //numPipes = 0 ;
> if (isPipe (command)) {
> splitPipes (command) ;
> for (i = 0; i < numPipes - 1; i++) {
> if (!pipeCmds (i, i + 1))
> printf ("Pipe failed\n") ;
> else
> printf ("Pipe succeeded\n") ;
>
> }
> }
> printf ("Exit\n") ;
> }
>
> int pipeCmds (int cmd1, int cmd2) {
> FILE *pipe_in ;
> FILE *pipe_out ;
> char *my_string ;
> int nbytes = 100;
> int bytes_read;
> char readbuf[MAX_CMD_LEN];
>
> printf ("Piping [%i] %s -> [%i] %s\n", cmd1, piped_cmds [cmd1], cmd2,
> piped_cmds [cmd2]) ;
> // open pipes
> pipe_in = popen (piped_cmds [cmd1], "r") ;
> printf ("Input pipe open\n") ;
> pipe_out = popen (piped_cmds [cmd2], "w") ;
> printf ("Output pipe open\n") ;
>
> if ((!pipe_in) || (!pipe_out)) {
> fprintf (stderr, "One or both pipes failed.\n") ;
> return 0 ;
> }
>
> printf ("Transferring file contents...\n") ;
> // while(fgets(readbuf, 250, pipe_in))
> // fputs(readbuf, pipe_out);
> while (!feof (pipe_in)) {
> printf ("Reading input pipe...\n") ;
> /****************program hangs here**************************/
> fgets (readbuf, MAX_CMD_LEN, pipe_in);
> if(feof(pipe_in))
> break;
> printf("%s", readbuf) ;
> fputs (readbuf, pipe_out) ;
> }
>
> pclose (pipe_in) ;
> pclose (pipe_out) ;
> printf ("Pipes closed\n") ;
>
> return 1 ;
>
>
> }
>
> char *stripFirstChar (char *str) {
> str++ ;
> return str ;
> }
>
> char *stripLastChar (char *str) {
> char *temp = str ;
> while (*temp != '\0') {
> temp++ ;
> }
> temp-- ;
> *temp = '\0' ;
> //printf ("\"%s\"\n", temp) ;
>
>
> return str ;
> }
>
> void splitPipes (char *toSplit) {
> char *tokenPtr = NULL ;
> int i ;
>
> for (tokenPtr = strtok (toSplit, "->"); tokenPtr != NULL;
> tokenPtr = strtok (NULL, "->"))
> {
> strcpy (piped_cmds [numPipes], tokenPtr) ;
> numPipes++ ;
> }
>
> for (i = 1; i < numPipes; i++) {
> strcpy (piped_cmds [i], stripFirstChar (piped_cmds [i])) ;
> }
>
> for (i = 0; i < numPipes - 1; i++) {
> strcpy (piped_cmds [i], stripLastChar (piped_cmds [i])) ;
> }
>
>
>
> for (i = 0; i < numPipes; i++) {
> printf ("[%i] \"%s\"\n", i, piped_cmds [i]) ;
> }
> }
>
> int isPipe (char *cmd) {
>
> if (strchr (cmd, '>')) {
> printf ("%s contains at least 1 pipe\n", cmd) ;
> return 1 ;
> }
> else
> printf ("%s contains no pipes\n", cmd) ;
> return 0 ;
> }

--
Barry Margolin, barmar@alum.mit.edu
Arlington, MA
*** PLEASE post questions in newsgroups, not directly to me ***

Jens.Toerring@physik.fu-berlin.de

2004-09-29, 3:09 am

In comp.unix.programmer Greg <djbitchpimp@snowboard.com> wrote:
> I am trying to implement the UNIX pipe command using C but with the
> "->" operator. Everything works fine with 1 pipe, but when I try to
> use 2 or more, it hangs up when reading the pipe_in filestream. If

<snipped code because of its length>

What you do in your program for A -> B -> C is basically

1. create a new process running A
2. create a new process running B
3. read input from first process, running A
4. write what you just read to second process, running B
5. kill both processes

and then

6. create a new process running B
7. create a new process running C
8. read input from third process, running B
9. write what you read to fourth process, running C
10. kill both processes

So its roughly as if you would write on the command line

> A | B; B | C

And, of courrse, that's not a pipe at all. There are two processes
running B that are completely independent of each other. But with
a pipe you would have only a single one in the middle, getting the
output of the process running A and its output being fed to the
third process. running C. So your program can't work because the
second instance of a process running B has nothing to output since
it never gets any input - that went to the first B process and the
output of that was simply thrown away.

You can't implement what you try to do with jsut popen(). At least
for B you need to manually fork() and exec() a new process with its
standard input _and_ output redirected to be able to both send data
to it and read them simultaneously (popen() allows this only for
either stdin or stdout redirected, but not both).

Another thing with your program that's not working is your use of
feof(). This function only returns something meaningful _after_
you have read from a FILE*, i.e. its purpose is to find out why
a previous read failed (or returned less than expected). You can't
use it to figure out if the next read would fail because of EOF,
that's not what it's meant for and so it must not be called before
you have tried to read something.
Regards, Jens
--
\ Jens Thoms Toerring ___ Jens.Toerring@physik.fu-berlin.de
\__________________________ http://www.toerring.de

Greg

2004-09-29, 8:09 pm

Jens.Toerring@physik.fu-berlin.de wrote in message news:<2rvbgrF1f58m8U1@uni-berlin.de>...
> In comp.unix.programmer Greg <djbitchpimp@snowboard.com> wrote:
>
> <snipped code because of its length>
>
> What you do in your program for A -> B -> C is basically
>
> 1. create a new process running A
> 2. create a new process running B
> 3. read input from first process, running A
> 4. write what you just read to second process, running B
> 5. kill both processes
>
> and then
>
> 6. create a new process running B
> 7. create a new process running C
> 8. read input from third process, running B
> 9. write what you read to fourth process, running C
> 10. kill both processes
>
> So its roughly as if you would write on the command line
>
>
> And, of courrse, that's not a pipe at all. There are two processes
> running B that are completely independent of each other. But with
> a pipe you would have only a single one in the middle, getting the
> output of the process running A and its output being fed to the
> third process. running C. So your program can't work because the
> second instance of a process running B has nothing to output since
> it never gets any input - that went to the first B process and the
> output of that was simply thrown away.
>
> You can't implement what you try to do with jsut popen(). At least
> for B you need to manually fork() and exec() a new process with its
> standard input _and_ output redirected to be able to both send data
> to it and read them simultaneously (popen() allows this only for
> either stdin or stdout redirected, but not both).
>
> Another thing with your program that's not working is your use of
> feof(). This function only returns something meaningful _after_
> you have read from a FILE*, i.e. its purpose is to find out why
> a previous read failed (or returned less than expected). You can't
> use it to figure out if the next read would fail because of EOF,
> that's not what it's meant for and so it must not be called before
> you have tried to read something.
> Regards, Jens

Thanks for your quick response!

Would I be able to store the output of the process B to a temporary
FILE * variable and return that, and then call the function
redirecting this temporary variable into the next process in the pipe?

Or would an alternative be to write the function differently for the
2nd, 3rd, and 4th pipes using the temporary FILE *?

Till Crueger

2004-09-29, 8:09 pm

On Wed, 29 Sep 2004 11:43:44 +0000, Greg wrote:

> Thanks for your quick response!
>
> Would I be able to store the output of the process B to a temporary
> FILE * variable and return that, and then call the function
> redirecting this temporary variable into the next process in the pipe?
>
> Or would an alternative be to write the function differently for the
> 2nd, 3rd, and 4th pipes using the temporary FILE *?

Maybe the best way to do it would be recursevly. so if you have A -> B ->
C you parse it as (A -> B) -> C. That means when someone calls your
Programm as A -> B -> C it will call itself with A -> B and then pass the
result of this proces to C. This recursion makes it a lot simpler,
although you'll have to spawn a lot of processes when the chain becomes
longer.

Till
--
Please add "Salt and Peper" to the subject line to bypass my spam filter

Jens.Toerring@physik.fu-berlin.de

2004-09-29, 8:09 pm

In comp.unix.programmer Greg <djbitchpimp@snowboard.com> wrote:
> Would I be able to store the output of the process B to a temporary
> FILE * variable and return that, and then call the function
> redirecting this temporary variable into the next process in the pipe?

The problem is that, as long as you insist on using popen() you
won't get at the output from B. popen() only allows to redirect
either stdin or stdout, so if you use popen() for sending data
into the B you have no way to get at the data it outputs and that
you would have to pass on to C. Redirecting the output of B
into a temporary file and then start C with its input redirected
to that file is only a rather awkward solution. It would be
basically be following command in the shell

A | B > tmp_file; C < tmp_file; rm tmp_file

And you would have to create another temporary file for each
further process in the chain of programs. So why not using real
pipes the OS is supplying you with?

The basic steps are

1. Create a pipe, using pipe(2)
2. fork(2) and in the child process close the read end side of
the pipe and redirect STDOUT_FILENO to the write end side,
using e.f. dup2(2).
3. exec(2) the program A in the child.

Back to the parent process:

4. Close the write end side of the pipe
5. Create another pipe
6. fork() and in the new child redirect STDIN_FILENO to the read
of the first pipe. Close the read end of the second pipe and
redirect STDOUT_FILENO to its write end.
7. exec(2) the program B in the new child.

And again back to the parent process:

8. Close the read end of the first and the write end side of the
second pipe
7. Create another child using fork.
8. In the child redirect STDIN_FILENO to the read end side of the
second pipe.
9. exec(2) the program C in the new child.

And, finally again in the parent

10. close the read end side of the second pipe.

That's more or less what the shell does when you create pipes
from the command line. What is done in the first and the third
section is basically what popen() can do for you. What you don't
get from popen() is what happens in the middle section. And if
you need more than 1 process in the middle you have to repeat
that section for each of the processes, i.e. if you want to have

A | B | C | D | E

you need that middle section for B, C and D.

Regards, Jens
--
\ Jens Thoms Toerring ___ Jens.Toerring@physik.fu-berlin.de
\__________________________ http://www.toerring.de

Martin Ambuhl

2004-09-29, 8:09 pm

Jens.Toerring@physik.fu-berlin.de wrote an off-topic response to an
off-topic question in clc:

[snip]

Please keep your postings on this subject in comp.unix.programmer and
out of comp.lang.c. You *do* have an obligation not to perpetuate
conversations that are off-topic in newsgroups to which you post.

[follow-ups set]

Martin Ambuhl

2004-09-29, 8:09 pm

Greg wrote wrote an off-topic response to an off-topic question in clc:

[snip]

Please keep your postings on this subject in comp.unix.programmer and
out of comp.lang.c. You *do* have an obligation not to perpetuate
conversations that are off-topic in newsgroups to which you post.

[follow-ups set]